Source code
// The neutrality audit. Each test has a pass mark fixed in the rule book
// before this version ran on live outlets.
const R = require('./rules');
const { spearman, median } = require('./scale');
// Severity points on each side of the partisan list.
function lexiconBalance() {
const side = { L: { count: 0, points: 0 }, R: { count: 0, points: 0 } };
for (const [, , , s] of R.partisanEntries()) { side[s].count++; side[s].points += R.PARTISAN_SEVERITY; }
const gap = Math.abs(side.L.points - side.R.points) / Math.max(side.L.points, side.R.points);
return { side, gap, pass: gap <= R.TESTS.lexiconBalance.tolerance };
}
function mirrorCheck() {
const missing = R.MIRROR_PAIRS.filter(([a, b]) => !a || !b);
return { pairs: R.MIRROR_PAIRS.length, missing, pass: missing.length === 0 };
}
// Name swap results across every analyzed report that has one.
// Reports containing a published exception label are left out.
function nameSwap(features) {
const excepted = (f) => [...f.head.words, ...f.body.words].some((h) => R.SWAP_EXCEPTION_TERMS.includes(h.t));
const tested = features.filter((f) => typeof f.swapGap === 'number' && !excepted(f));
const gaps = tested.map((f) => f.swapGap);
if (!gaps.length) return { n: 0, pass: null };
const mean = gaps.reduce((a, b) => a + b, 0) / gaps.length;
const worst = tested.filter((f) => f.swapGap > R.TESTS.nameSwap.singleMax);
const max = Math.max(...gaps);
return { n: gaps.length, mean, max, worst: worst.slice(0, 10), pass: mean <= R.TESTS.nameSwap.meanMax && worst.length === 0 };
}
function anchorParity(anchors) {
const p = anchors.parity;
if (!p) return { pass: null };
const g = (a, b) => Math.abs(a - b) / Math.max(a, b, 1e-9);
const gap = g(p.leftWords, p.rightWords);
const tempGap = p.leftTemp != null ? g(p.leftTemp, p.rightTemp) : 0;
return { ...p, gap, tempGap, pass: gap <= R.TESTS.anchorParity.tolerance && tempGap <= R.TESTS.anchorParity.tolerance };
}
// Reranks outlets under alternative weights and combining rules, and reports
// how closely each ranking matches the default (Spearman correlation).
function sensitivity(outletArticles, scoreFn, geometric, arithmetic) {
const variants = {
'Default (geometric mean, published weights)': { combine: geometric, weights: R.WEIGHTS },
'Equal weights (geometric mean)': { combine: geometric, weights: { words: 0.25, temperature: 0.25, sourcing: 0.25, headline: 0.25 } },
'Published weights (simple average)': { combine: arithmetic, weights: R.WEIGHTS },
'Word choice only': { combine: geometric, weights: { words: 1 } },
};
const results = {};
for (const [name, v] of Object.entries(variants)) {
const means = {};
for (const [o, list] of Object.entries(outletArticles)) {
if (list.length < R.CONSTANTS.outletFloorReports) continue;
const s = list.map((f) => scoreFn(f, v).overall);
means[o] = s.reduce((a, b) => a + b, 0) / s.length;
}
results[name] = means;
}
const base = results['Default (geometric mean, published weights)'];
const rows = Object.entries(results).map(([name, means]) => ({ name, rho: spearman(base, means), outlets: Object.keys(means).length }));
const others = rows.slice(1).filter((r) => r.rho != null);
return { rows, pass: others.length ? others.every((r) => r.rho >= R.TESTS.sensitivity.min) : null };
}
// Krippendorff's alpha for nominal labels. units: [[label, label, ...], ...]
// α = 1 − observed disagreement ÷ disagreement expected by chance
function krippendorffNominal(units) {
const usable = units.filter((u) => u.length >= 2);
const n = usable.reduce((s, u) => s + u.length, 0);
if (n < 4) return null;
const counts = {};
for (const u of usable) for (const v of u) counts[v] = (counts[v] || 0) + 1;
let Do = 0;
for (const u of usable) {
const m = u.length;
const c = {};
for (const v of u) c[v] = (c[v] || 0) + 1;
let pairs = 0;
for (const a of Object.keys(c)) for (const b of Object.keys(c)) if (a !== b) pairs += c[a] * c[b];
Do += pairs / (m - 1);
}
Do /= n;
let De = 0;
for (const a of Object.keys(counts)) for (const b of Object.keys(counts)) if (a !== b) De += counts[a] * counts[b];
De /= n * (n - 1);
return De === 0 ? 1 : 1 - Do / De;
}
// Mantel-Haenszel differential item functioning for one word.
// strata: [{ leftFlag, leftNo, rightFlag, rightNo }] matched on human-rated loadedness.
// Δ = −2.35 × ln(α_MH)
function mantelHaenszel(strata) {
let num = 0; let den = 0;
for (const s of strata) {
const t = s.leftFlag + s.leftNo + s.rightFlag + s.rightNo;
if (!t) continue;
num += (s.leftFlag * s.rightNo) / t;
den += (s.leftNo * s.rightFlag) / t;
}
if (!num || !den) return null;
const alpha = num / den;
const delta = -2.35 * Math.log(alpha);
const a = Math.abs(delta);
return { alpha, delta, verdict: a < R.TESTS.dif.review ? 'pass' : a < R.TESTS.dif.remove ? 'review' : 'revise or remove' };
}
// Rasch model fit by joint maximum likelihood: P(flag) = exp(β − δ) ÷ (1 + exp(β − δ)).
// rows: excerpts, cols: words, values 1 (raters called it loaded), 0, or null.
function rasch(matrix, iters = 50) {
const P = matrix.length; const I = P ? matrix[0].length : 0;
if (P < 5 || I < 2) return null;
const beta = new Array(P).fill(0); const delta = new Array(I).fill(0);
const p = (b, d) => 1 / (1 + Math.exp(-(b - d)));
for (let k = 0; k < iters; k++) {
for (let i = 0; i < P; i++) {
let g = 0; let h = 0;
for (let j = 0; j < I; j++) if (matrix[i][j] != null) { const q = p(beta[i], delta[j]); g += matrix[i][j] - q; h += q * (1 - q); }
if (h) beta[i] = Math.max(-5, Math.min(5, beta[i] + g / h));
}
for (let j = 0; j < I; j++) {
let g = 0; let h = 0;
for (let i = 0; i < P; i++) if (matrix[i][j] != null) { const q = p(beta[i], delta[j]); g += q - matrix[i][j]; h += q * (1 - q); }
if (h) delta[j] = Math.max(-5, Math.min(5, delta[j] + g / h));
}
const mean = delta.reduce((a, b) => a + b, 0) / I;
for (let j = 0; j < I; j++) delta[j] -= mean;
}
return { delta, beta };
}
// Turns panel ratings into agreement figures and per-word results.
// ratings: [{ excerptId, rater, group, terms: { term: 'loaded'|'plain'|'unsure' }, overall: 1..5, lean: 'L'|'C'|'R' (outlet lean, hidden from raters) }]
function panelReport(ratings) {
const byUnitTerm = {};
const byGroupUnitTerm = { left: {}, center: {}, right: {} };
for (const r of ratings) {
for (const [t, v] of Object.entries(r.terms || {})) {
if (v === 'unsure') continue;
const key = `${r.excerptId}|${t}`;
(byUnitTerm[key] ||= []).push(v);
((byGroupUnitTerm[r.group] ||= {})[key] ||= []).push(v);
}
}
const overall = krippendorffNominal(Object.values(byUnitTerm));
const within = {};
for (const [g, units] of Object.entries(byGroupUnitTerm)) within[g] = krippendorffNominal(Object.values(units));
// Between groups: one majority label per group per unit, then agreement across groups.
const between = [];
for (const key of Object.keys(byUnitTerm)) {
const labels = [];
for (const g of ['left', 'center', 'right']) {
const v = (byGroupUnitTerm[g] || {})[key];
if (v && v.length) labels.push(majority(v));
}
if (labels.length >= 2) between.push(labels);
}
// Per-word verdicts: share of raters calling it loaded, by group.
const words = {};
for (const r of ratings) for (const [t, v] of Object.entries(r.terms || {})) {
if (v === 'unsure') continue;
const w = (words[t] ||= { t, n: 0, loaded: 0, byGroup: {} });
w.n++; if (v === 'loaded') w.loaded++;
const g = (w.byGroup[r.group] ||= { n: 0, loaded: 0 }); g.n++; if (v === 'loaded') g.loaded++;
}
return { raters: new Set(ratings.map((r) => r.rater)).size, ratings: ratings.length, overall, within, between: krippendorffNominal(between), words: Object.values(words).sort((a, b) => b.n - a.n) };
}
function majority(list) {
const c = {};
for (const v of list) c[v] = (c[v] || 0) + 1;
return Object.entries(c).sort((a, b) => b[1] - a[1])[0][0];
}
// The section of the outlet's site a report came from, read from its link.
const GENRES = [['Courts and crime', /\/(courts?|law|legal|justice|crime|supreme-court)\//i], ['World', /\/(world|international|europe|asia|africa|middle-east|americas|ukraine|israel)\//i], ['Business and economy', /\/(business|economy|money|markets|finance|tech|technology)\//i], ['Science, health and climate', /\/(science|health|climate|environment|weather)\//i], ['Politics', /\/(politics|elections?|congress|white-house|policy|government|us-news|national)\//i]];
function genreOf(link) {
let path = '';
try { path = new URL(link).pathname; } catch { path = String(link || ''); }
for (const [g, re] of GENRES) if (re.test(path)) return g;
return 'Other or unlabeled';
}
// Stability check on held-out events: reports are split in two by event (so
// coverage of one event never sits in both halves), and the same summaries
// are computed on each half. Results that hold in both halves are less likely
// to come from a few unusual events.
function stability(saved, scoreFn, groupIntoEvents, leans) {
const reports = Object.values(saved).filter((f) => f.published && !f.opinion && !f.wire);
const events = groupIntoEvents(reports.map((f) => ({ title: f.title, link: f.link, summary: '', published: f.published, outletId: f.outletId, id: f.id })));
const half = {};
for (const e of events) {
let h = 0;
for (const c of e.id) h = (h * 31 + c.charCodeAt(0)) >>> 0;
for (const a of e.articles) half[a.id] = h % 2 ? 'held-out' : 'development';
}
const out = { development: { n: 0, byLean: {}, byGenre: {} }, 'held-out': { n: 0, byLean: {}, byGenre: {} } };
for (const f of reports) {
const s = scoreFn(f);
if (!s.scored) continue;
const part = out[half[f.id] || 'development'];
part.n++;
const lean = (leans[f.outletId] || {}).lean || 'C';
(part.byLean[lean] ||= []).push(s.overall);
(part.byGenre[genreOf(f.link)] ||= []).push(s.overall);
}
const avg = (x) => (x && x.length ? x.reduce((a, b) => a + b, 0) / x.length : null);
for (const p of Object.values(out)) {
p.lean = Object.fromEntries(Object.entries(p.byLean).map(([k, v]) => [k, { mean: avg(v), n: v.length }]));
p.genre = Object.fromEntries(Object.entries(p.byGenre).map(([k, v]) => [k, { mean: avg(v), n: v.length }]));
delete p.byLean; delete p.byGenre;
}
return out;
}
// Pairs of reports for checking the grouping by hand: pairs the method put in
// the same event, and near misses it kept apart.
function groupingPairs(saved, similarity, fingerprint, threshold = 0.5, days = 7) {
const list = Object.values(saved).filter((f) => f.title && f.published).sort((a, b) => a.id.localeCompare(b.id));
const cutoff = Date.now() - days * 86400000;
const recent = list.filter((f) => Date.parse(f.published) >= cutoff);
const fps = recent.map((f) => ({ f, fp: fingerprint({ title: f.title, summary: '' }) }));
const same = []; const near = [];
for (let i = 0; i < fps.length; i++) for (let j = i + 1; j < fps.length; j++) {
const a = fps[i]; const b = fps[j];
if (a.f.outletId === b.f.outletId) continue;
if (Math.abs(Date.parse(a.f.published) - Date.parse(b.f.published)) > 48 * 3600 * 1000) continue;
const sim = similarity(a.fp, b.fp);
const row = { a: a.f.id, b: b.f.id, ta: a.f.title, tb: b.f.title, sim: Math.round(sim * 100) / 100 };
if (sim >= threshold) same.push(row); else if (sim >= 0.2) near.push(row);
}
const pickEvery = (arr, n) => (arr.length <= n ? arr : Array.from({ length: n }, (_, k) => arr[Math.floor((k * arr.length) / n)]));
return [...pickEvery(same, 100), ...pickEvery(near, 100)];
}
// Grouping error rates from hand-labeled pairs. Each label says whether two
// headlines report the same event; the method's decision is recomputed from
// the stored headlines with the current rule.
function groupingErrors(labels, similarity, fingerprint, threshold = 0.5) {
let tp = 0; let fp = 0; let fn = 0; let tn = 0;
for (const l of labels || []) {
const sim = similarity(fingerprint({ title: l.ta, summary: '' }), fingerprint({ title: l.tb, summary: '' }));
const said = sim >= threshold;
if (said && l.same) tp++; else if (said && !l.same) fp++; else if (!said && l.same) fn++; else tn++;
}
const n = tp + fp + fn + tn;
if (!n) return null;
return { n, tp, fp, fn, tn, precision: tp + fp ? tp / (tp + fp) : null, recall: tp + fn ? tp / (tp + fn) : null };
}
// Panel agreement per word group (by the newsroom rule each word enforces).
function agreementByRule(ratings, ruleOf) {
const byRule = {};
for (const r of ratings) for (const [t, v] of Object.entries(r.terms || {})) {
if (v === 'unsure') continue;
const rule = ruleOf(t) || 'other';
((byRule[rule] ||= {})[`${r.excerptId}|${t}`] ||= []).push(v);
}
return Object.fromEntries(Object.entries(byRule).map(([k, units]) => [k, { alpha: krippendorffNominal(Object.values(units)), units: Object.keys(units).length }]));
}
module.exports = { genreOf, stability, groupingPairs, groupingErrors, agreementByRule, lexiconBalance, mirrorCheck, nameSwap, anchorParity, sensitivity, krippendorffNominal, mantelHaenszel, rasch, panelReport, median };