Source code

// Candidate words for the lists, for people to review (3.1.0). Never adds a
// word and never changes a score. Compares word use in opinion pieces with
// news reports, over the privately kept text, by the weighted log-odds
// method with an informative Dirichlet prior (Monroe, Colaresi & Quinn 2008):
//   δ = log((y_o + a_w) ÷ (n_o + a0 − y_o − a_w)) − log((y_n + a_w) ÷ (n_n + a0 − y_n − a_w))
//   z = δ ÷ √(1 ÷ (y_o + a_w) + 1 ÷ (y_n + a_w))
// where y is a word's count, n the total words in each group, a_w the word's
// count in both groups together (the prior) and a0 the total of all words.

const R = require('./rules');
const { ownVoice } = require('./text');
const { isOpinion } = require('./opinion');
const { listText, readTextPath } = require('./store');

const STOP = new Set('that this with from have were they their there which would about after said says will been when what more than into also over just like some only other could them then most such even many much very before while where these those does dont because being under each since until again still both between'.split(' '));

function tokens(text) {
  return (String(text).toLowerCase().match(/[a-z][a-z'-]{3,}/g) || []).filter((w) => !STOP.has(w));
}

async function candidates({ limit = 400, top = 60 } = {}) {
  const paths = (await listText(null, 5000)).slice(-limit);
  const groups = { opinion: new Map(), news: new Map() };
  const n = { opinion: 0, news: 0 };
  const docs = { opinion: 0, news: 0 };
  let i = 0;
  await Promise.all(Array.from({ length: 16 }, async () => {
    while (i < paths.length) {
      const t = await readTextPath(paths[i++]);
      if (!t || !t.paragraphs) continue;
      const g = isOpinion(t.link, t.categories || []) ? 'opinion' : 'news';
      docs[g]++;
      for (const w of tokens(ownVoice(t.paragraphs.join(' ')).text)) { groups[g].set(w, (groups[g].get(w) || 0) + 1); n[g]++; }
    }
  }));
  const listed = new Set([...R.WORD_CHOICE, ...R.HEAT, ...R.SOFT].map(([w]) => w.toLowerCase()).concat(R.MIRROR_PAIRS.flat().map((w) => w.toLowerCase())));
  const all = new Map();
  for (const g of Object.values(groups)) for (const [w, c] of g) all.set(w, (all.get(w) || 0) + c);
  const a0 = [...all.values()].reduce((a, b) => a + b, 0);
  const rows = [];
  for (const [w, aw] of all) {
    if (aw < 5 || listed.has(w)) continue;
    const yo = groups.opinion.get(w) || 0;
    const yn = groups.news.get(w) || 0;
    const d = Math.log((yo + aw) / (n.opinion + a0 - yo - aw)) - Math.log((yn + aw) / (n.news + a0 - yn - aw));
    const z = d / Math.sqrt(1 / (yo + aw) + 1 / (yn + aw));
    rows.push({ word: w, z: Math.round(z * 100) / 100, opinion: yo, news: yn });
  }
  rows.sort((a, b) => b.z - a.z);
  return { method: R.VERSION, texts: docs, words: n, note: 'Candidates for people to review. Positive z: used more in opinion pieces than in news reports. Not added to any list and not used in any score.', candidates: rows.slice(0, top) };
}

module.exports = { candidates, tokens };