Source code
// One function serves every page. vercel.json sends all paths here.
const crypto = require('crypto');
const { getData, build, backfill } = require('../lib/data');
const { readJson, writeJson } = require('../lib/store');
const { calibrate } = require('../lib/calibrate');
const L = require('../lib/leans');
const P = require('../lib/pages');
const P3 = require('../lib/pages3');
const CHECKS_PATH = 'readers/checks.json';
const SOURCE_FILES = ['api/index.js', 'lib/rules.js', 'lib/analyze.js', 'lib/text.js', 'lib/scale.js', 'lib/cluster.js', 'lib/data.js', 'lib/audit.js', 'lib/challenge.js', 'lib/steering.js', 'lib/opinion.js', 'lib/fetchtext.js', 'lib/panels.js', 'lib/records.js', 'lib/candidates.js', 'lib/store.js', 'lib/rss.js', 'lib/archive.js', 'lib/outlets.js', 'lib/leans.js', 'lib/evidence.json', 'lib/anchors.json', 'lib/grouping-labels.json'];
// Fonts are served from this site (so readers' browsers contact no font
// company). The function fetches each file once from the published Fontsource
// package and the edge cache keeps it for a year.
const FONTS = {
'plex-sans-400.woff2': 'ibm-plex-sans@5.2.5/files/ibm-plex-sans-latin-400-normal.woff2',
'plex-sans-600.woff2': 'ibm-plex-sans@5.2.5/files/ibm-plex-sans-latin-600-normal.woff2',
'plex-serif-600.woff2': 'ibm-plex-serif@5.2.5/files/ibm-plex-serif-latin-600-normal.woff2',
'plex-mono-400.woff2': 'ibm-plex-mono@5.2.5/files/ibm-plex-mono-latin-400-normal.woff2',
'OFL.txt': 'ibm-plex-sans@5.2.5/LICENSE',
};
async function font(res, name) {
const r = await fetch(`https://cdn.jsdelivr.net/npm/@fontsource/${FONTS[name]}`, { signal: AbortSignal.timeout(8000) });
if (!r.ok) { res.statusCode = 502; res.setHeader('cache-control', 'no-store'); return res.end(); }
res.statusCode = 200;
res.setHeader('content-type', name.endsWith('.txt') ? 'text/plain; charset=utf-8' : 'font/woff2');
res.setHeader('cache-control', 'public, max-age=31536000, s-maxage=31536000, immutable');
res.setHeader('access-control-allow-origin', '*');
return res.end(Buffer.from(await r.arrayBuffer()));
}
// Hand-labeled headline pairs for the grouping check. The file stores report
// ids; headlines come from the saved reports, so a pair drops out of the check
// once either report leaves the live window.
function labels(data) {
let pairs = [];
try { pairs = require('../lib/grouping-labels.json').pairs || []; } catch { return []; }
return pairs.map(([a, b, same]) => (data.saved[a] && data.saved[b] ? { ta: data.saved[a].title, tb: data.saved[b].title, same: Boolean(same) } : null)).filter(Boolean);
}
const LABEL_NOTE = ' The sample is half pairs the method joined and half near misses, so close calls are over-represented and these rates are not the rates across all coverage. Rule used for labeling: two headlines are the same event when they report the same specific happening within two days, such as the same debate, rally, court filing, announcement, or campaign ad and the immediate reaction to it. <a href="/source/lib/grouping-labels.json">The labels</a>.';
function text(res, status, body, type) { res.statusCode = status; res.setHeader('content-type', `${type}; charset=utf-8`); res.setHeader('cache-control', 'public, s-maxage=600'); res.end(body); }
const PANEL_PATH = 'panel/ratings.json';
const FEEDBACK_PATH = 'feedback/items.json';
function send(res, status, html, { cache = true } = {}) {
res.statusCode = status;
res.setHeader('content-type', 'text/html; charset=utf-8');
res.setHeader('x-content-type-options', 'nosniff');
res.setHeader('referrer-policy', 'strict-origin-when-cross-origin');
res.setHeader('cache-control', cache ? 'public, s-maxage=600, stale-while-revalidate=1800' : 'no-store');
res.end(html);
}
function json(res, status, value) {
res.statusCode = status;
res.setHeader('content-type', 'application/json; charset=utf-8');
res.setHeader('cache-control', 'no-store');
res.end(JSON.stringify(value, null, 2));
}
function redirect(res, to) { res.statusCode = 303; res.setHeader('location', to); res.setHeader('cache-control', 'no-store'); res.end(); }
async function body(req) {
if (req.body && typeof req.body === 'object') return req.body;
const raw = typeof req.body === 'string' ? req.body : await new Promise((ok) => { let d = ''; req.on('data', (c) => { d += c; if (d.length > 20000) req.destroy(); }); req.on('end', () => ok(d)); });
return Object.fromEntries(new URLSearchParams(raw));
}
// Panel codes come from the PANEL_CODES setting: "code:left,code:center,...".
function panelRater(code) {
if (!code) return null;
for (const entry of String(process.env.PANEL_CODES || '').split(',')) {
const [c, g] = entry.split(':').map((x) => (x || '').trim());
if (c && g && crypto.timingSafeEqual(Buffer.from(c.padEnd(64)), Buffer.from(String(code).padEnd(64).slice(0, 64))) && ['left', 'center', 'right'].includes(g)) return { code: c, group: g, id: crypto.createHash('sha256').update(c).digest('hex').slice(0, 10) };
}
return null;
}
const isAdmin = (code) => Boolean(process.env.ADMIN_CODE) && typeof code === 'string' && code.length === process.env.ADMIN_CODE.length && crypto.timingSafeEqual(Buffer.from(code), Buffer.from(process.env.ADMIN_CODE));
// A narrower code for the daily outlet-request check: it can read the request
// queue, test a feed, post outcomes and catch up a new outlet, and nothing else.
const isOps = (code) => isAdmin(code) || (Boolean(process.env.REQUESTS_CODE) && typeof code === 'string' && code.length === process.env.REQUESTS_CODE.length && crypto.timingSafeEqual(Buffer.from(code), Buffer.from(process.env.REQUESTS_CODE)));
// Picks the next passage for a rater, cycling through left, center and
// right outlets so the gold set stays balanced.
function nextExcerpt(data, ratings, rater) {
const done = new Set(ratings.filter((r) => r.rater === rater.id).map((r) => r.excerptId));
const pools = { L: [], C: [], R: [] };
for (const f of Object.values(data.saved)) {
(f.excerpts || []).forEach((x, i) => {
const id = `${f.id}:${i}`;
if (!done.has(id)) pools[(L.leans[f.outletId] || {}).lean || 'C'].push({ id, text: x.text, terms: x.terms, outletId: f.outletId });
});
}
const order = ['L', 'C', 'R'].sort((a, b) => ratings.filter((r) => r.rater === rater.id && r.lean === a).length - ratings.filter((r) => r.rater === rater.id && r.lean === b).length);
for (const k of order) if (pools[k].length) {
// Prefer passages others have already rated, so agreement can be measured.
const counts = {};
for (const r of ratings) counts[r.excerptId] = (counts[r.excerptId] || 0) + 1;
pools[k].sort((a, b) => (counts[b.id] || 0) - (counts[a.id] || 0) || a.id.localeCompare(b.id));
return pools[k][0];
}
return null;
}
module.exports = async (req, res) => {
const url = new URL(req.url, 'http://x');
const path = (url.searchParams.get('__p') != null ? `/${url.searchParams.get('__p')}` : url.pathname).replace(/\/+$/, '') || '/';
const q = url.searchParams;
try {
if (path === '/api/calibrate') {
if (!isAdmin(q.get('code'))) return json(res, 403, { error: 'admin code required' });
const data = await build();
const result = await calibrate(Object.values(data.saved));
try { result.archivedAs = await require('../lib/archive').saveCalibration(result); } catch (err) { console.error('calibration archive failed', err.message); }
return json(res, 200, result);
}
if (path === '/api/refresh') {
const auth = req.headers && req.headers.authorization;
if (!isAdmin(q.get('code')) && !(process.env.CRON_SECRET && auth === `Bearer ${process.env.CRON_SECRET}`)) return json(res, 403, { error: 'not allowed' });
const d = await build();
const reasons = {};
for (const f of Object.values(d.saved)) { const r = (reasons[f.outletId] ||= { page: 0 }); if (f.source === 'page') r.page++; else if (f.pageNote) r[f.pageNote] = (r[f.pageNote] || 0) + 1; else r.waiting = (r.waiting || 0) + 1; }
return json(res, 200, { builtAt: d.builtAt, pagesRead: d.pagesRead, pagesPending: d.pagesPending, rescored: d.rescored, held: d.held, textStore: d.textStore, reasons });
}
// Candidate words for people to review, from the privately kept text.
if (path === '/api/candidates') {
if (!isAdmin(q.get('code'))) return json(res, 403, { error: 'admin code required' });
return json(res, 200, await require('../lib/candidates').candidates({ limit: Math.min(2000, Number(q.get('limit')) || 400) }));
}
// Deletes every kept text for one outlet, at the outlet's request.
if (path === '/api/text/delete') {
if (!isAdmin(q.get('code'))) return json(res, 403, { error: 'admin code required' });
const outlet = String(q.get('outlet') || '').replace(/[^a-z0-9-]/gi, '');
if (!outlet) return json(res, 400, { error: 'outlet required' });
const removed = await require('../lib/store').deleteTextFor(outlet);
return json(res, 200, { outlet, removed });
}
if (path === '/api/backfill') {
if (!isOps(q.get('code'))) return json(res, 403, { error: 'code required' });
return json(res, 200, await backfill(String(q.get('outlet') || '')));
}
if (path === '/api/requests') {
if (!isOps(q.get('code'))) return json(res, 403, { error: 'code required' });
const items = await readJson(FEEDBACK_PATH, []);
return json(res, 200, items.filter((x) => x.kind === 'Add an outlet' && !x.outcome).map(({ id, link, text, name, at }) => ({ id, link, text, name, at })));
}
if (path === '/api/requests/resolve') {
if (!isOps(q.get('code'))) return json(res, 403, { error: 'code required' });
const items = await readJson(FEEDBACK_PATH, []);
const it = items.find((x) => x.id === q.get('id'));
if (!it) return json(res, 404, { error: 'no such request' });
it.outcome = String(q.get('outcome') || '').slice(0, 1000);
it.reviewedAt = new Date().toISOString();
await writeJson(FEEDBACK_PATH, items);
return json(res, 200, { id: it.id, outcome: it.outcome });
}
if (path === '/api/vet') {
// Tests a feed the way the site reads it: from this server, with this
// site's reader and robots.txt rules.
if (!isOps(q.get('code'))) return json(res, 403, { error: 'code required' });
const feed = String(q.get('feed') || '');
if (!/^https?:\/\//.test(feed)) return json(res, 400, { error: 'feed must be a full web address' });
const { fetchOutlet } = require('../lib/rss');
const { fetchPage } = require('../lib/fetchtext');
const { extractParagraphs, countWords } = require('../lib/text');
const r = await fetchOutlet({ id: 'vet', name: 'vet', feed });
const out = { feed, feedOk: r.ok, feedError: r.error || null, items: r.articles.length, dated: r.articles.filter((a) => a.published).length, newest: r.articles.map((a) => a.published).filter(Boolean).sort().pop() || null, samples: [] };
for (const a of r.articles.slice(0, 3)) {
const p = await fetchPage(a.link);
const paras = p.ok ? extractParagraphs(p.html) : [];
out.samples.push({ title: a.title, link: a.link, pageOk: p.ok, reason: p.ok ? null : p.reason, words: countWords(paras.join(' ')) });
}
return json(res, 200, out);
}
if (path === '/api/headlines') {
if (!isAdmin(q.get('code'))) return json(res, 403, { error: 'admin code required' });
// Headline-only figures per outlet, for comparing outlets whose article
// text cannot be read.
const d = await getData();
const out = {};
for (const f of Object.values(d.saved)) {
const o = (out[f.outletId] ||= { headlines: 0, flagged: 0, points: 0, heatPoints: 0, words: 0 });
const hits = [...(f.head.words || []), ...(f.head.up || []), ...(f.head.down || [])];
o.headlines++; o.words += f.headWords || 0;
if (hits.length) o.flagged++;
o.points += hits.reduce((n, h) => n + h.s, 0);
o.heatPoints += [...(f.head.up || []), ...(f.head.down || [])].reduce((n, h) => n + h.s, 0);
}
for (const o of Object.values(out)) { o.flaggedShare = +(o.flagged / o.headlines).toFixed(3); o.pointsPer1000 = +((o.points / Math.max(1, o.words)) * 1000).toFixed(1); }
return json(res, 200, out);
}
if (path.startsWith('/fonts/') && FONTS[path.slice(7)]) return font(res, path.slice(7));
if (path === '/api/pairs') {
if (!isAdmin(q.get('code'))) return json(res, 403, { error: 'admin code required' });
const d = await getData();
const { similarity, fingerprint } = require('../lib/cluster');
return json(res, 200, require('../lib/audit').groupingPairs(d.saved, similarity, fingerprint));
}
if (path === '/method') return send(res, 200, require('../lib/methodpage').methodPage());
if (path === '/about') return send(res, 200, require('../lib/about').aboutPage(await getData()));
if (path === '/archive/score-trend.json') return json(res, 200, await require('../lib/archive').readTrend());
if (path === '/changelog') return redirect(res, '/changes');
if (path === '/changes') return send(res, 200, P3.changesPage(await readJson(FEEDBACK_PATH, []), await readJson('grouping/overrides.json', { merges: [], splits: [] })), { cache: false });
if (path === '/research') return send(res, 200, P3.researchPage());
if (path === '/source') return send(res, 200, P3.sourceIndex(SOURCE_FILES));
if (path.startsWith('/source/')) {
const name = path.slice(8);
if (!SOURCE_FILES.includes(name)) return send(res, 404, P.notFound(), { cache: false });
const file = require('path').join(__dirname, '..', name);
return send(res, 200, P3.sourceFile(name, require('fs').readFileSync(file, 'utf8')));
}
if (path === '/check') {
if (req.method === 'POST') {
const b = await body(req);
const answers = P3.CHECK_QUESTIONS.map((_, i) => (b[`q${i}`] == null ? null : Number(b[`q${i}`])));
const checks = await readJson(CHECKS_PATH, []);
if (checks.length < 20000) { checks.push({ answers, at: new Date().toISOString() }); await writeJson(CHECKS_PATH, checks); }
const right = answers.filter((a, i) => a === P3.CHECK_QUESTIONS[i].right).length;
return redirect(res, `/check?done=${right}`);
}
const done = q.get('done');
return send(res, 200, P3.checkPage({ done: done != null, score: Number(done) || 0 }), { cache: false });
}
if (path === '/archive') return send(res, 200, require('../lib/archivepage').archivePage(await require('../lib/archive').listArchive()), { cache: false });
if (/^\/archive\/(reports\/\d{4}-\d{2}|reports\/undated|scores\/\d{4}-\d{2}-\d{2}|placement\/\d{4}-\d{2}-\d{2}|calibrations\/[\w-]+)\.json$/.test(path)) {
const file = path.slice(1);
const value = file.startsWith('archive/reports/') ? await require('../lib/archive').readMerged(file) : await readJson(file, null);
if (!value || (typeof value === 'object' && !Object.keys(value).length)) return json(res, 404, { error: 'not found' });
res.setHeader('content-disposition', `inline; filename="${file.split('/').pop()}"`);
return json(res, 200, value);
}
if (path === '/evidence') return send(res, 200, require('../lib/evidence').evidencePage());
if (path === '/request') return send(res, 200, require('../lib/request').requestPage({ items: await readJson(FEEDBACK_PATH, []), message: q.get('sent') ? 'Thank you. Your request is saved and will be checked within a day.' : '' }), { cache: false });
if (path === '/panel') {
const ratings = await readJson(PANEL_PATH, []);
if (req.method === 'POST') {
const b = await body(req);
const rater = panelRater(b.code);
if (!rater) return send(res, 403, P.panelPage({ needCode: true, message: 'That code was not recognized.' }), { cache: false });
const terms = {};
for (let i = 0; i < 12; i++) if (b[`t${i}`] && ['loaded', 'plain', 'unsure'].includes(b[`v${i}`])) terms[String(b[`t${i}`]).slice(0, 80)] = b[`v${i}`];
const overall = Math.min(5, Math.max(1, parseInt(b.overall, 10) || 3));
const fid = String(b.excerpt || '').split(':')[0];
const data = await getData();
const f = data.saved[fid];
if (f && !ratings.some((r) => r.rater === rater.id && r.excerptId === b.excerpt)) {
ratings.push({ excerptId: String(b.excerpt), rater: rater.id, group: rater.group, terms, overall, lean: (L.leans[f.outletId] || {}).lean || 'C', at: new Date().toISOString() });
await writeJson(PANEL_PATH, ratings);
}
return redirect(res, `/panel?code=${encodeURIComponent(rater.code)}&saved=1`);
}
const rater = panelRater(q.get('code'));
if (!rater) return send(res, 200, P.panelPage({ needCode: true, message: q.get('code') ? 'That code was not recognized.' : '' }), { cache: false });
const data = await getData();
return send(res, 200, P.panelPage({ rater, excerpt: nextExcerpt(data, ratings, rater), message: q.get('saved') ? 'Saved.' : '' }), { cache: false });
}
if (path === '/feedback') {
const items = await readJson(FEEDBACK_PATH, []);
if (req.method === 'POST') {
const b = await body(req);
if (b.admin) {
if (!isAdmin(b.admin)) return send(res, 403, P3.feedbackPage(items, { message: 'Admin code not recognized.' }), { cache: false });
const it = items.find((x) => x.id === b.id);
if (it) {
it.outcome = String(b.outcome || '').slice(0, 1000);
it.status = P3.STATUSES.includes(b.status) ? b.status : 'Answered';
it.reviewedAt = new Date().toISOString();
// The score when reviewed, so readers can see whether a fix moved it.
try { const d = await getData(); const f = Object.values(d.saved).find((x) => x.link === it.link); if (f) it.scoreAtReview = d.scored[f.id].s.overall; } catch { /* the outcome is saved without it */ }
const OV = 'grouping/overrides.json';
if (b.merge || b.split) {
const ov = await readJson(OV, { merges: [], splits: [] });
const [from, into] = String(b.merge || '').split('>').map((x) => x.trim());
if (from && into) ov.merges.push({ from, into, at: it.reviewedAt, dispute: it.id });
if (String(b.split || '').trim()) ov.splits.push({ report: String(b.split).trim(), at: it.reviewedAt, dispute: it.id });
await writeJson(OV, ov);
}
await writeJson(FEEDBACK_PATH, items);
}
return redirect(res, `/feedback?admin=${encodeURIComponent(b.admin)}`);
}
const link = String(b.link || '').slice(0, 500);
const txt = String(b.text || '').slice(0, 3000);
if (!/^https?:\/\//.test(link) || txt.trim().length < 5) return send(res, 400, P3.feedbackPage(items, { message: 'Add a link and a short explanation.' }), { cache: false });
if (items.length > 5000) return send(res, 429, P3.feedbackPage(items, { message: 'The queue is full for now. Please try again later.' }), { cache: false });
const kind = P3.DISPUTE_KINDS.includes(b.kind) ? b.kind : 'A score looks wrong';
items.push({ id: crypto.randomUUID(), link, kind, text: txt, name: String(b.name || '').slice(0, 120), at: new Date().toISOString() });
await writeJson(FEEDBACK_PATH, items);
return redirect(res, kind === 'Add an outlet' ? '/request?sent=1' : '/feedback?sent=1');
}
const admin = isAdmin(q.get('admin')) ? q.get('admin') : '';
return send(res, 200, P3.feedbackPage(items, { link: q.get('link') || '', kind: q.get('kind') || '', message: q.get('sent') ? 'Thank you. Your submission is saved and will get a published outcome once reviewed.' : '', admin }), { cache: false });
}
const data = await getData();
const SP = require('../lib/steeringpages');
if (path === '/') return send(res, 200, P3.feedPage(data));
if (path === '/outlets') return send(res, 200, P3.outletsPage(data));
if (path === '/status') return send(res, 200, P3.statusPage(data));
if (path === '/search') return send(res, 200, P3.searchPage(data, q.get('q')), { cache: false });
if (path === '/steering') return send(res, 200, SP.steeringPage(data));
if (path === '/method/card') return send(res, 200, P.cardPage(data));
if (path === '/audit') return send(res, 200, P3.auditPage(data, await readJson(PANEL_PATH, []), { checks: await readJson(CHECKS_PATH, []), labels: labels(data) }).replace('and are pending human review.</p>', `and are pending human review.${LABEL_NOTE}</p>`));
let m = path.match(/^\/e\/([a-z0-9]+)\.(json|csv)$/);
if (m && data.eventById[m[1]]) {
const ev = data.eventById[m[1]];
res.setHeader('content-disposition', `attachment; filename="transparency-news-event-${ev.id}.${m[2]}"`);
return text(res, 200, P3.eventDownload(ev, data, m[2]), m[2] === 'json' ? 'application/json' : 'text/csv');
}
m = path.match(/^\/e\/([a-z0-9]+)$/);
// Steering sections are added to the event and outlet pages here.
if (m && data.eventById[m[1]]) return send(res, 200, P3.eventPage(data.eventById[m[1]], data).replace('<section><h2>Timeline</h2>', `${SP.eventSteering(data.eventById[m[1]], data)}<section><h2>Timeline</h2>`));
m = path.match(/^\/o\/([a-z0-9]+)$/);
if (m && data.outletById[m[1]]) return send(res, 200, P3.outletPage(data.outletById[m[1]], data, await require('../lib/archive').readTrend()).replace('<section><h2>Recent reports</h2>', `${SP.outletSteering(data.outletById[m[1]], data)}<section><h2>Recent reports</h2>`));
m = path.match(/^\/a\/([a-f0-9]{12})$/);
if (m && data.saved[m[1]]) { const f = data.saved[m[1]]; const disputes = (await readJson(FEEDBACK_PATH, [])).filter((x) => x.link === f.link); return send(res, 200, P3.articlePage(f, data, disputes)); }
return send(res, 404, P.notFound(), { cache: false });
} catch (err) {
console.error(err);
return send(res, 500, '<!doctype html><title>Transparency News</title><p>The page could not be built just now. Try again in a minute.</p>', { cache: false });
}
};