Source code
// Measures one report. The published record holds counts, matched words, up
// to three short sentences for the human rating panel and the few words
// around each flagged word. Since 3.1.0 the text of a page read with the
// outlet's permission is also kept, privately and never published, so the
// report can be measured again after a rule change (see lib/store.js).
const crypto = require('crypto');
const R = require('./rules');
const { countWords, ownVoice, sentences } = require('./text');
const ST = require('./steering');
const P = require('./panels');
const EXTRACTOR_VERSION = '4';
function escapeRe(s) { return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); }
// One matcher per layer. Longer phrases are tried first, so "woke mob" counts
// once as a partisan label and not again as "mob".
function matcher(entries) {
const map = new Map(entries.map((e) => [e[0].toLowerCase(), e]));
const alts = [...map.keys()].sort((a, b) => b.length - a.length).map(escapeRe);
const re = new RegExp(`(?<![a-z\\-])(${alts.join('|')})(?![a-z\\-])`, 'gi');
return { re, map };
}
const M = {
words: matcher([...R.WORD_CHOICE, ...R.partisanEntries()]),
heat: matcher(R.HEAT),
soft: matcher(R.SOFT),
unnamed: matcher(R.UNNAMED.map((w) => [w, 2, 'unnamed'])),
vague: matcher(R.VAGUE.map((w) => [w, 2, 'vague'])),
};
const NEG_RE = new RegExp(`\\b${R.CONTEXT.negation}\\b`, 'i');
const LITERAL = Object.fromEntries(Object.entries(R.CONTEXT.literal).filter(([, v]) => v).map(([k, v]) => [k, new RegExp(`^\\s+(?:${v})\\b`, 'i')]));
// The sentence around a position in a text.
function sentenceAt(text, at) {
const start = Math.max(text.lastIndexOf('. ', at), text.lastIndexOf('? ', at), text.lastIndexOf('! ', at)) + 1;
let end = text.slice(at).search(/[.?!](\s|$)/);
end = end < 0 ? text.length : at + end + 1;
return text.slice(start, end).trim();
}
// How the reporter is using a matched word (3.0.0 context rules).
function classify(text, at, term, rule) {
const before = text.slice(Math.max(0, at - 80), at);
const after = text.slice(at + term.length, at + term.length + 60);
const lastWords = before.split(/\s+/).filter(Boolean).slice(-R.CONTEXT.negationWindow).join(' ');
if (NEG_RE.test(lastWords)) return 'negated';
if (R.CONTEXT.discussing.test(before)) return 'discussed';
const lit = LITERAL[term.toLowerCase()];
if (lit && lit.test(after)) return 'literal';
if (!R.CONTEXT.attributionExempt.includes(rule)) {
// Unnamed and vague sources ("sources said", "critics say") do not
// excuse the reporter's own wording, so they are removed before the check.
const sent = sentenceAt(text, at).replace(M.unnamed.re, ' ').replace(M.vague.re, ' ');
const withoutTerm = sent.replace(new RegExp(escapeRe(term), 'i'), ' ');
if (R.CONTEXT.attribution.test(withoutTerm)) return 'attributed';
}
return 'reporter';
}
// A few words either side of a match, so a reader can judge the context.
function snippet(text, at, len) {
const left = text.slice(Math.max(0, at - 60), at).split(/\s+/).slice(-6).join(' ');
const right = text.slice(at + len, at + len + 60).split(/\s+/).slice(0, 7).join(' ');
return `${left} [${text.slice(at, at + len)}] ${right}`.replace(/\s+/g, ' ').trim().slice(0, 140);
}
function find(m, text, { context = true } = {}) {
const out = [];
for (const hit of String(text).matchAll(m.re)) {
const e = m.map.get(hit[1].toLowerCase());
const sev = R.effectiveSeverity(e[0], e[1], e[2]);
out.push({
term: e[0], sev: sev.s, listed: e[1], rule: e[2], side: e[3] || null, at: hit.index,
x: sev.experimental, cls: context ? classify(text, hit.index, hit[1], e[2]) : 'reporter',
ctx: context ? snippet(text, hit.index, hit[1].length) : null,
});
}
return out;
}
// Sourcing hits: a stated reason for anonymity in the same sentence lowers
// the severity from 2 to the published REASON_POINTS (0 since 3.0.0).
function sourcing(text) {
const hits = [];
let cursor = 0;
for (const s of sentences(text).length ? sentences(text) : [text]) {
// Where the sentence starts in the text, so each hit keeps its position.
const off = Math.max(0, text.indexOf(s, cursor));
cursor = off + s.length;
const reason = R.REASON_GIVEN.test(s);
for (const h of find(M.unnamed, s)) hits.push({ ...h, at: h.at + off, sev: reason ? R.REASON_POINTS : 2, listed: 2, reason });
for (const h of find(M.vague, s)) hits.push({ ...h, at: h.at + off });
}
return hits;
}
// Word number of each hit (0 for the first word of the text), so the densest
// 200-word stretch of a story can be found (3.1.0).
function addWordPositions(text, m) {
const starts = [];
for (const w of String(text).matchAll(/[A-Za-z0-9][A-Za-z0-9'’\-]*/g)) starts.push(w.index);
const wordAt = (at) => { let lo = 0; let hi = starts.length; while (lo < hi) { const mid = (lo + hi) >> 1; if (starts[mid] <= at) lo = mid + 1; else hi = mid; } return Math.max(0, lo - 1); };
for (const k of Object.keys(m)) for (const h of m[k]) h.w = wordAt(h.at);
return m;
}
// Measures a block of the outlet's own words.
function measure(text) {
return { words: find(M.words, text), up: find(M.heat, text), down: find(M.soft, text), src: sourcing(text) };
}
function swapNames(text) {
let out = text;
const pairs = R.SWAP_PAIRS;
const tokens = [];
pairs.forEach(([a, b], i) => {
out = out.replace(new RegExp(`\\b${escapeRe(a)}\\b`, 'g'), `\u0000A${i}\u0000`);
out = out.replace(new RegExp(`\\b${escapeRe(b)}\\b`, 'g'), `\u0000B${i}\u0000`);
tokens.push([a, b]);
});
return out.replace(/\u0000([AB])(\d+)\u0000/g, (_, side, i) => (side === 'A' ? tokens[i][1] : tokens[i][0]));
}
// Names and numbers mentioned in the report, for the fact coverage view.
function factTokens(text) {
const facts = new Map();
for (const m of text.matchAll(/\b\d[\d,.]*\s?(?:%|percent|million|billion|trillion|votes?|years?|people|days)?/g)) {
const t = m[0].trim();
if (t.length > 1 && !/^(19|20)\d\d$/.test(t)) facts.set(t.toLowerCase(), t);
}
for (const m of text.matchAll(/\b(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,2})\b/g)) {
if (!/^(The|A|An|In|On|At|But|And|He|She|They|It|This|That|When|While|After|Before)\s/.test(m[0])) facts.set(m[0].toLowerCase(), m[0]);
}
return [...facts.values()].slice(0, 60);
}
function articleId(link) {
return crypto.createHash('sha1').update(link).digest('hex').slice(0, 12);
}
function wireOf(text) {
const head = String(text || '').slice(0, 400);
for (const [name, re] of R.WIRE) if (re.test(head)) return name;
return null;
}
// Names and figures in the headline that also appear in the story: a rough,
// experimental check of whether the headline's specifics are supported.
function headlineSupport(title, bodyRaw) {
const tokens = factTokens(title || '').filter((t) => t.length > 2);
if (!tokens.length) return null;
const body = String(bodyRaw || '').toLowerCase();
const found = tokens.filter((t) => body.includes(t.toLowerCase()));
return { tokens: tokens.slice(0, 8), found: found.length };
}
function archiveLink(link, iso) {
const stamp = iso.replace(/[-:T]/g, '').slice(0, 14);
return `https://web.archive.org/web/${stamp}/${link}`;
}
// article: { title, link, summary, published, outletId }; paragraphs: [] when
// the page could not be read; page: { links, table } from the page, if read.
function analyze(article, paragraphs, scoreFn, page = {}) {
const fromPage = paragraphs && paragraphs.length > 0;
const bodyRaw = fromPage ? paragraphs.join(' ') : article.summary || '';
const head = ownVoice(article.title || '');
const body = ownVoice(bodyRaw);
const ownWords = countWords(body.text);
const headWords = countWords(head.text);
const hm = measure(head.text);
const bm = addWordPositions(body.text, measure(body.text));
// The opening: the first 80 words of the story's own text, for the
// headline-versus-opening comparison (shown, not scored).
const openText = body.text.split(/\s+/).slice(0, R.CONSTANTS.openingWords).join(' ');
const om = measure(openText);
// Loaded words inside quotations: shown beside the score, never in it.
const qText = [...head.quotes, ...body.quotes].join(' . ');
const qm = qText ? { words: find(M.words, qText, { context: false }), up: find(M.heat, qText, { context: false }), down: find(M.soft, qText, { context: false }) } : { words: [], up: [], down: [] };
const attrib = (bodyRaw.match(R.ATTRIBUTION_MARKERS) || []).length;
const now = new Date().toISOString();
// Up to three sentences that carry a flag, for the blind human panel.
const excerpts = [];
for (const s of sentences(body.text)) {
if (excerpts.length >= 3) break;
const m = measure(s);
const terms = [...m.words, ...m.up, ...m.down, ...m.src].map((h) => h.term);
if (terms.length && countWords(s) <= 45) excerpts.push({ text: blind(s), terms: [...new Set(terms)] });
}
const pts = (hits) => hits.filter((h) => h.cls === 'reporter' && !h.x).reduce((n, h) => n + h.sev, 0);
const f = {
id: articleId(article.link),
outletId: article.outletId,
link: article.link,
title: article.title,
published: article.published,
source: fromPage ? 'page' : 'feed',
ownWords,
headWords,
quotedWords: body.quotedWords + head.quotedWords,
headQuoted: head.quotedWords,
head: compact(hm),
body: compact(bm),
open: { words: countWords(openText), points: pts([...om.words, ...om.up, ...om.down]) },
quoted: { words: body.quotedWords + head.quotedWords, hits: [...qm.words, ...qm.up, ...qm.down].map((h) => ({ t: h.term, s: h.listed, r: h.rule })) },
scare: [...head.scare, ...body.scare].slice(0, 10),
attrib,
wire: wireOf(bodyRaw) || wireOf(article.summary),
headSupport: headlineSupport(article.title, bodyRaw),
facts: factTokens(`${article.title}. ${bodyRaw}`),
excerpts,
analyzedAt: now,
rulesVersion: R.VERSION,
lexiconFp: R.LEXICON_FP,
// What was read, so a later reader can tell whether a result can be reproduced.
capture: { at: now, extractor: EXTRACTOR_VERSION, complete: fromPage, textFp: crypto.createHash('sha256').update(body.text).digest('hex').slice(0, 16), archive: archiveLink(article.link, now) },
// Steering signals, shown beside the score and never part of it.
frames: ST.framePhrases(head.text, body.text),
numbers: ST.numberFrames(body.text, sentences),
firstFigure: ST.firstFigure(body.text, head.text),
fv: ST.FEATURES_V,
// Panels shown beside the score, never part of it (3.1.0).
check: P.checkability({ bodyRaw, links: page.links || [], table: page.table, srcHits: bm.src.map((h) => ({ r: h.rule })), attrib }),
structure: P.structure(bodyRaw),
tallies: P.tallies(bodyRaw),
bills: P.bills(`${article.title || ''}. ${bodyRaw}`),
};
// Name-swap test: score the same report with party, politician and
// institution names swapped, and record how far the overall score moved.
if (scoreFn) {
const sw = analyze({ ...article, title: swapNames(article.title || ''), summary: swapNames(article.summary || '') }, fromPage ? paragraphs.map(swapNames) : [], null, page);
f.swapGap = Math.abs(scoreFn(f).overall - scoreFn(sw).overall);
// Structure panel check: swapping the names must swap the two sides' counts.
f.structureFlip = JSON.stringify(sw.structure.A) === JSON.stringify(f.structure.B) && JSON.stringify(sw.structure.B) === JSON.stringify(f.structure.A);
}
return f;
}
// Stored form of each hit: term, scored severity, listed severity, rule,
// context class, experimental flag, side (partisan pairs), reason given
// (sourcing), and the words around it.
function compact(m) {
const pick = (hits) => hits.map((h) => ({ t: h.term, s: h.sev, s0: h.listed, r: h.rule, ...(h.cls && h.cls !== 'reporter' ? { c: h.cls } : {}), ...(h.x ? { x: 1 } : {}), ...(h.side ? { side: h.side } : {}), ...(h.reason ? { reason: 1 } : {}), ...(h.w != null ? { w: h.w } : {}), ...(h.ctx ? { ctx: h.ctx } : {}) }));
return { words: pick(m.words), up: pick(m.up), down: pick(m.down), src: pick(m.src) };
}
// Removes outlet names from excerpts so raters cannot tell the source.
const OUTLET_NAMES = /\b(NPR|PBS|BBC|New York Times|Times|Guardian|Politico|The Hill|Axios|CBS|ABC|NBC|HuffPost|Fox News|Fox|New York Post|Post|Washington Times|Washington Examiner|Examiner|Daily Caller|Breitbart|Federalist|Vox|Tucker Carlson)\b/g;
function blind(s) { return s.replace(OUTLET_NAMES, '[outlet]'); }
module.exports = { analyze, measure, swapNames, factTokens, articleId, M, find, classify, EXTRACTOR_VERSION };