Source code

// Three panels shown beside the score and never part of it (3.1.0):
// checkability (what a reader could check), structure (how each party is
// described in sentences that mention only that party), and the raw material
// for record checks (vote tallies and bill numbers).

const R = require('./rules');
const { ownVoice, sentences, countWords } = require('./text');

// ---------- Checkability ----------

const NAME = "[A-Z][a-z]+(?:[ -][A-Z][a-z'’]+){1,2}";
const SPEAK = '(?:said|says|told|wrote|testified|added|explained|noted)';
const NAMED_AFTER = new RegExp(`\\b(${NAME})(?:,[^,.]{1,70},)?\\s${SPEAK}\\b`, 'g');
const NAMED_BEFORE = new RegExp(`\\b${SPEAK}\\s(${NAME})\\b`, 'g');
const NOT_A_PERSON = /^(The|This|That|These|Those|White House|Supreme Court|Justice Department|State Department|United States|New York|Associated Press|In|On|But|And)\b/;

const PRIMARY = /(\.gov|\.mil)(\/|$)|congress\.gov|uscourts|supremecourt|courtlistener|documentcloud|federalregister|regulations\.gov|\.pdf(\?|#|$)|sec\.gov|census\.gov|bls\.gov|fec\.gov/i;
const DOC_CITE = /\b(?:according to|cited in|in)\s(?:the|a|an)\s(?:\w+\s){0,3}(?:report|filing|complaint|indictment|lawsuit|bill|ruling|order|opinion|memo|memorandum|transcript|data|records?|audit|study|survey|budget|statement|letter)\b|\b(?:court|public|financial|campaign finance|budget)\s(?:records|documents|filings|data)\b|\bthe\s(?:bill|complaint|ruling|filing|order|report|indictment)\s(?:says|said|states|stated|shows|showed|reads)\b/gi;
const CORRECTION = /\b(?:Correction|Clarification|Editor's note)\s*:|An earlier version of this (?:story|article|report)|This (?:story|article|report) has been (?:corrected|updated to correct)/i;
const NUMBER = /\b\d[\d,.]*\b/g;
const MONTH = /\b(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan\.|Feb\.|Aug\.|Sept\.|Oct\.|Nov\.|Dec\.)\s\d{1,2}\b/g;

function namedSources(text) {
  const names = new Set();
  for (const re of [NAMED_AFTER, NAMED_BEFORE]) for (const m of String(text).matchAll(re)) if (!NOT_A_PERSON.test(m[1])) names.add(m[1]);
  return [...names];
}

// Outbound links in the article block of the page, for primary-record links.
function articleLinks(html) {
  if (!html) return [];
  const art = html.match(/<article[\s>][\s\S]*?<\/article>/i);
  const block = art && art[0].length > 2000 ? art[0] : html;
  const out = new Set();
  for (const m of block.matchAll(/<a\s[^>]*href="(https?:\/\/[^"#]+)"/gi)) out.add(m[1]);
  return [...out].slice(0, 300);
}
function hasDataTable(html) { return /<table[\s>]/i.test(html || '') || /datawrapper|flourish\.studio|infogram/i.test(html || ''); }

// bodyRaw: the text read, quotes included; links: outbound article links;
// srcHits: the flagged sourcing phrases already found; attrib: attribution markers.
function checkability({ bodyRaw, links = [], table = false, srcHits = [], attrib = 0 }) {
  const named = namedSources(bodyRaw);
  const unnamed = srcHits.filter((h) => h.r === 'unnamed').length;
  const primary = [...new Set(links.filter((u) => PRIMARY.test(u)))];
  const docs = (String(bodyRaw).match(DOC_CITE) || []).length;
  const specifics = (String(bodyRaw).match(NUMBER) || []).length + (String(bodyRaw).match(MONTH) || []).length;
  const citesNoOne = named.length === 0 && unnamed === 0 && attrib === 0 && primary.length === 0 && docs === 0;
  return {
    named: named.length, namedList: named.slice(0, 12), unnamed,
    unnamedShare: named.length + unnamed ? unnamed / (named.length + unnamed) : null,
    primary: primary.length, primaryList: primary.slice(0, 8), docs, specifics,
    table: Boolean(table), correction: CORRECTION.test(bodyRaw), citesNoOne,
  };
}

// ---------- Record-check material ----------

const TALLY = /\b(\d{1,3})\s?(?:-|–|—|to)\s?(\d{1,3})\b/g;
const VOTE_WORDS = /\b(?:vote|voted|votes|passed|confirmed|approved|rejected|failed|roll call|cloture)\b/i;
function tallies(text) {
  const out = [];
  for (const s of sentences(text)) {
    if (!VOTE_WORDS.test(s)) continue;
    const chamber = /\bSenate\b/.test(s) ? 'Senate' : /\bHouse\b/.test(s) ? 'House' : null;
    for (const m of s.matchAll(TALLY)) {
      const a = +m[1]; const b = +m[2];
      const max = chamber === 'Senate' ? 100 : 435;
      if (a + b < 20 || a + b > max || /\d{4}/.test(m[0])) continue;
      out.push({ yea: Math.max(a, b), nay: Math.min(a, b), chamber, text: m[0] });
    }
  }
  const seen = new Set();
  return out.filter((t) => { const k = `${t.chamber}|${t.yea}|${t.nay}`; return seen.has(k) ? false : (seen.add(k), true); }).slice(0, 6);
}

const BILL = /\b(H\.\s?R\.|H\.\s?Res\.|H\.\s?J\.\s?Res\.|S\.\s?J\.\s?Res\.|S\.\s?Res\.|S\.)\s?(\d{1,5})\b/g;
const BILL_TYPE = (t) => t.replace(/[\s.]/g, '').toLowerCase();
function bills(text) {
  const out = new Map();
  for (const m of String(text).matchAll(BILL)) {
    const type = BILL_TYPE(m[1]);
    if (type === 's' && +m[2] < 10) continue; // "S. 1" style false matches on initials are rare; very low numbers are skipped
    out.set(`${type}${m[2]}`, { type, number: +m[2], text: m[0] });
  }
  return [...out.values()].slice(0, 6);
}

// ---------- Structure, per party ----------

// Sides come from the published name-swap pairs: the first name of each pair
// is side A (Democratic), the second side B (Republican). Because the swap
// test exchanges exactly these names, swapping names must flip every
// asymmetry's sign.
const PARTY_PAIRS = R.SWAP_PAIRS.filter(([a]) => /^(Democratic Party|Democrats|Democratic|Democrat|Schumer|Jeffries|Pelosi|Newsom|Sanders|Ocasio-Cortez|Walz|Harris)$/.test(a));
const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const SIDE_RE = {
  A: new RegExp(`\\b(?:${PARTY_PAIRS.map(([a]) => esc(a)).join('|')})\\b`),
  B: new RegExp(`\\b(?:${PARTY_PAIRS.map(([, b]) => esc(b)).join('|')})\\b`),
};
const SIDE_NAMES = { A: 'Democratic side', B: 'Republican side' };
const PASSIVE = /\b(?:was|were|is|are|been|being)\s+(?:\w+ly\s+)?\w+(?:ed|en)\b/i;
const LOADED_VERB = /\b(?:claimed|claims|claim|admitted|admits|insisted|insists|conceded|concedes|alleged|alleges|boasted|boasts|complained|complains|railed|rails|lashed out|bragged|brags)\b/i;
const HEDGE = /\b(?:reportedly|allegedly|apparently|seemingly|purportedly|supposedly|so-called)\b/i;
const TITLE = /\b(?:President|Vice President|Sen\.|Senator|Rep\.|Representative|Gov\.|Governor|Speaker|Leader|Secretary|Mayor)\s+[A-Z]/;
const MEASURES = ['passive', 'hidden', 'loadedVerb', 'hedge', 'scare', 'title'];
const MEASURE_NAMES = {
  passive: 'Passive voice', hidden: 'Passive voice with no one named as doing it', loadedVerb: 'Loaded attribution verbs (claimed, admitted, insisted)',
  hedge: 'Hedges (reportedly, allegedly, so-called)', scare: 'Scare quotes', title: 'Titles used (Sen., Gov., President)',
};

function blank() { return { m: 0, passive: 0, hidden: 0, loadedVerb: 0, hedge: 0, scare: 0, title: 0, quoted: 0 }; }

function structure(bodyRaw) {
  const out = { A: blank(), B: blank(), first: null, last: null };
  for (const s of sentences(bodyRaw)) {
    const a = SIDE_RE.A.test(s); const b = SIDE_RE.B.test(s);
    if (a === b) continue; // neither side, or both: not counted
    const side = a ? 'A' : 'B';
    const c = out[side];
    const own = ownVoice(s);
    c.m++;
    if (PASSIVE.test(own.text)) { c.passive++; if (!/\bby\b/i.test(own.text)) c.hidden++; }
    if (LOADED_VERB.test(own.text)) c.loadedVerb++;
    if (HEDGE.test(own.text)) c.hedge++;
    if (own.scare.length) c.scare++;
    if (TITLE.test(s)) c.title++;
    c.quoted += own.quotedWords;
    if (own.quotedWords >= 4) { if (!out.first) out.first = side; out.last = side; }
  }
  return out;
}

// Asymmetry of one measure between the sides: A = (pA − pB) ÷ (pA + pB),
// with p = (count + 1) ÷ (opportunities + 2). Positive means side A gets more.
function asym(xA, mA, xB, mB) {
  const pA = (xA + 1) / (mA + 2);
  const pB = (xB + 1) / (mB + 2);
  return (pA - pB) / (pA + pB);
}

// Adds up per-report structure counts (an event or an outlet) and returns the
// asymmetries, shown only when each side has at least 10 sentences.
const MIN_OPPORTUNITIES = 10;
const MIN_COUNT = 3; // an asymmetry is shown only when the two sides together have 3 or more cases
function summarize(list) {
  const A = blank(); const B = blank();
  let firstA = 0; let firstB = 0; let lastA = 0; let lastB = 0;
  for (const st of list.filter(Boolean)) {
    for (const k of Object.keys(A)) { A[k] += st.A[k] || 0; B[k] += st.B[k] || 0; }
    if (st.first === 'A') firstA++; if (st.first === 'B') firstB++;
    if (st.last === 'A') lastA++; if (st.last === 'B') lastB++;
  }
  const enough = A.m >= MIN_OPPORTUNITIES && B.m >= MIN_OPPORTUNITIES;
  const rows = MEASURES.map((k) => ({ k, name: MEASURE_NAMES[k], a: A[k], b: B[k], asym: enough && A[k] + B[k] >= MIN_COUNT ? asym(A[k], A.m, B[k], B.m) : null }));
  const voice = A.quoted + B.quoted ? A.quoted / (A.quoted + B.quoted) : null;
  return { A, B, enough, rows, voiceShareA: voice, first: { A: firstA, B: firstB }, last: { A: lastA, B: lastB }, reports: list.filter(Boolean).length };
}

const sideNames = (i) => [...new Set(PARTY_PAIRS.map((p) => p[i]))];
module.exports = { sideNames, MIN_COUNT, checkability, articleLinks, hasDataTable, tallies, bills, structure, summarize, asym, namedSources, PARTY_PAIRS, SIDE_NAMES, MEASURE_NAMES, MIN_OPPORTUNITIES, countWords };