Source code
// Steering signals: ways a report can shape what readers want without saying
// so. These are tracked and shown beside the plainness score; they never
// change it. Each signal is tied to published research on the Steering page.
//
// Per report (measured when the page is read):
// frames short phrases from the headline and opening (repetition uses the headline), for spotting a
// frame an outlet repeats far more than other outlets do
// numbers sentences that pair a figure with gain words or loss words
// firstFigure the first figure in the outlet's own words
// sponsor a sponsor or paid-content label, its wording and its place
// Across reports (worked out at each update):
// repetition phrases one outlet repeats much more than the others
// coverage stories most outlets carried that one outlet's feed did not
// placement which saved reports sit highest on each outlet's front page
const FEATURES_V = 2;
const STOP = new Set(('a an and are as at be been but by for from has have had he her his how i in into is it its of on or our over she so than that the their them they this to up was we were what when where which who why will with would you your after again all also any about against before being between both did does doing down during each few further here just more most new not now off once only other out own same says say said should some such then there these those through too under until very while amid back can could may might must one two three first last year years week day today tuesday wednesday thursday friday saturday sunday monday news live update updates report reports video watch per via get gets got make makes made take takes took like s t don didn isn wasn doesn mr ms mrs')
.split(' '));
const tokens = (text) => String(text || '').toLowerCase().replace(/[’']/g, "'").split(/[^a-z0-9'\-]+/).map((w) => w.replace(/^['\-]+|['\-]+$/g, '').replace(/'s$/, '')).filter(Boolean);
// Two- and three-word phrases of content words from the headline and the
// first 60 words of the outlet's own text. Up to 30 per report.
function framePhrases(headText, bodyText) {
const lead = String(bodyText || '').split(/\s+/).slice(0, 60).join(' ');
const out = new Set();
for (const part of [headText, lead]) {
const w = tokens(part);
for (let i = 0; i < w.length; i++) {
for (const n of [2, 3]) {
const g = w.slice(i, i + n);
if (g.length < n) continue;
if (g.some((t) => STOP.has(t) || /^\d/.test(t) || t.length < 3)) continue;
out.add(g.join(' '));
}
}
}
return [...out].slice(0, 30);
}
// Gain and loss words that sit in the same sentence as a figure. The same
// outcome can be told as lives saved or lives lost, jobs added or jobs cut.
const GAIN = ['saved', 'saves', 'save', 'saving', 'survive', 'survived', 'survives', 'protect', 'protected', 'protects', 'gain', 'gained', 'gains', 'benefit', 'benefits', 'benefited', 'created', 'creates', 'added', 'adds', 'boost', 'boosted', 'boosts', 'won', 'wins'];
const LOSS = ['lost', 'lose', 'loses', 'losing', 'killed', 'kill', 'kills', 'died', 'die', 'dies', 'deaths', 'dead', 'cut', 'cuts', 'slashed', 'slashes', 'eliminated', 'eliminates', 'shed', 'destroyed', 'wiped out', 'stripped'];
const FIGURE = /(\$\s?\d[\d,.]*(?:\s?(?:trillion|billion|million|thousand))?|\d[\d,.]*\s?(?:%|percent|trillion|billion|million|thousand)|\b\d{1,3}(?:,\d{3})+\b|\b\d+(?:\.\d+)?\b(?=\s+(?:people|jobs|lives|workers|families|children|patients|households|students|voters|deaths|cases|troops|homes)))/i;
const wordRe = (list) => new RegExp(`\\b(${list.map((w) => w.replace(/ /g, '\\s+')).join('|')})\\b`, 'i');
const GAIN_RE = wordRe(GAIN);
const LOSS_RE = wordRe(LOSS);
function isYear(s) { return /^(1[89]|20)\d\d$/.test(String(s).trim()); }
function numberFrames(text, sentences) {
const res = { gain: 0, loss: 0, examples: [] };
for (const s of sentences(text)) {
const fig = s.match(FIGURE);
if (!fig || isYear(fig[1])) continue;
const g = s.match(GAIN_RE);
const l = s.match(LOSS_RE);
if (g) res.gain++;
if (l) res.loss++;
if ((g || l) && res.examples.length < 3) res.examples.push({ figure: fig[1].trim().slice(0, 30), word: (l || g)[1].toLowerCase(), frame: l && !g ? 'loss' : g && !l ? 'gain' : 'both' });
}
return res;
}
function firstFigure(bodyText, headText) {
for (const t of [bodyText, headText]) {
const re = new RegExp(FIGURE.source, 'gi');
let m;
while ((m = re.exec(String(t || '')))) if (!isYear(m[1])) return m[1].trim().slice(0, 40);
}
return null;
}
// Sponsor labels. Ad widgets labeled "Sponsored" appear on most news pages,
// so only these count: the link's own path, the page's own section metadata,
// the outlet's feed category, or a label within the headline area.
const SPONSOR_PATH = /\/(sponsored|sponsor-content|sponsored-content|paid-post|paidpost|paid-content|partner-content|brandvoice|brand-studio|brandstudio|native|advertorial|promoted)(\/|-|$)/i;
// A bare "Advertisement" is the label on ordinary ad slots, so it does not count.
const EXPLICIT = /\b(paid advertisement|paid post|paid content|sponsored content|sponsored by|advertorial|paid partnership)\b/i;
// Feed categories and section names that mark paid content (not stories about advertising).
const PAID_SECTION = /^(sponsored|sponsor content|sponsored content|paid post|paid content|partner content|advertorial|brand ?voice|brand studio|promoted|native)$/i;
const VAGUE = /\b(presented by|in partnership with|partner content|brand ?voice|brought to you by|promoted content|from our partners?|special advertising section)\b/i;
function sponsorFrom({ html, link, categories = [] }) {
const found = [];
try { if (SPONSOR_PATH.test(new URL(link).pathname)) found.push({ where: 'link', label: new URL(link).pathname.match(SPONSOR_PATH)[1] }); } catch { /* bad link */ }
for (const c of categories || []) if (PAID_SECTION.test(String(c).trim())) found.push({ where: 'feed category', label: String(c).slice(0, 40) });
const h = String(html || '');
const section = h.match(/<meta[^>]+(?:property|name)=["'](?:article:section|parsely-section|sailthru\.tags|category)["'][^>]+content=["']([^"']{1,80})["']/i)
|| h.match(/"articleSection"\s*:\s*"([^"]{1,80})"/i);
if (section && PAID_SECTION.test(section[1].trim())) found.push({ where: 'page section', label: section[1] });
const h1 = h.search(/<h1[\s>]/i);
if (h1 >= 0) {
const near = h.slice(Math.max(0, h1 - 1500), h1 + 2500).replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ');
const e = near.match(EXPLICIT) || near.match(VAGUE);
if (e) found.push({ where: 'near headline', label: e[1] });
}
if (!found.length) return null;
const labels = found.map((x) => x.label).join(' ');
return { wording: EXPLICIT.test(labels) || found.some((x) => x.where === 'link' && /paid|sponsor|advertorial/i.test(x.label)) ? 'explicit' : 'vague', where: [...new Set(found.map((x) => x.where))], labels: [...new Set(found.map((x) => x.label.toLowerCase()))].slice(0, 3) };
}
// ---- Across reports ----
const DAY = 86400000;
// Phrases an outlet uses in at least 3 reports and at least 4 times as often
// (by share of its reports) as all other outlets combined.
function repetition(saved, outlets, { days = 30, now = Date.now() } = {}) {
// Headlines only: article openings carry boilerplate (wire credits, video captions).
const recent = Object.values(saved).filter((f) => f.title && Date.parse(f.published) >= now - days * DAY && !f.opinion).map((f) => ({ ...f, frames: framePhrases(f.title, '') }));
const per = {}; const tot = {}; const n = {}; let all = 0;
for (const f of recent) {
all++; n[f.outletId] = (n[f.outletId] || 0) + 1;
const p = (per[f.outletId] ||= {});
for (const ph of new Set(f.frames)) { p[ph] = (p[ph] || 0) + 1; tot[ph] = (tot[ph] || 0) + 1; }
}
const out = {};
for (const o of outlets) {
const p = per[o.id] || {}; const mine = n[o.id] || 0;
const others = all - mine;
const rows = [];
for (const [ph, c] of Object.entries(p)) {
if (c < 3 || mine < 5) continue;
const shareMine = c / mine;
const shareOthers = ((tot[ph] - c) + 0.5) / (others + 1);
const lift = shareMine / shareOthers;
if (lift >= 4) rows.push({ phrase: ph, count: c, reports: mine, shareMine, shareOthers: (tot[ph] - c) / Math.max(1, others), lift });
}
// Drop a two-word phrase when a three-word phrase containing it has the same count.
rows.sort((a, b) => b.count - a.count || b.lift - a.lift);
const keep = rows.filter((r) => !rows.some((x) => x !== r && x.phrase.includes(r.phrase) && x.count === r.count));
out[o.id] = { reports: mine, phrases: keep.slice(0, 12) };
}
return { days, out };
}
function numberFraming(saved, outlets, { days = 30, now = Date.now() } = {}) {
const out = {};
for (const o of outlets) out[o.id] = { reports: 0, gain: 0, loss: 0 };
for (const f of Object.values(saved)) {
if (!f.numbers || !out[f.outletId] || f.opinion || Date.parse(f.published) < now - days * DAY) continue;
const r = out[f.outletId]; r.reports++; r.gain += f.numbers.gain; r.loss += f.numbers.loss;
}
return out;
}
// Applies the current label rule to labels saved under an earlier rule.
function stillPaid(sp) {
if (!sp) return false;
return sp.labels.some((l) => EXPLICIT.test(l) || VAGUE.test(l) || PAID_SECTION.test(l.trim()) || (sp.where.includes('link') && SPONSOR_PATH.test(`/${l}/`)));
}
function sponsors(saved, outlets) {
const out = {};
for (const o of outlets) out[o.id] = { reports: 0, labeled: [] };
for (const f of Object.values(saved)) {
if (!out[f.outletId] || f.fv == null) continue;
out[f.outletId].reports++;
if (stillPaid(f.sponsor)) out[f.outletId].labeled.push({ id: f.id, title: f.title, sponsor: f.sponsor });
}
return out;
}
// Stories carried by at least 40% of active outlets (and at least 4) in the
// last 14 days, and which active outlets' feeds did not carry them.
function coverage(saved, outlets, groupIntoEvents, { days = 14, now = Date.now() } = {}) {
const recent = Object.values(saved).filter((f) => f.published && Date.parse(f.published) >= now - days * DAY).map((f) => ({ title: f.title, link: f.link, summary: '', published: f.published, outletId: f.outletId }));
const active = outlets.filter((o) => recent.some((a) => a.outletId === o.id)).map((o) => o.id);
const need = Math.max(4, Math.ceil(active.length * 0.4));
const wide = groupIntoEvents(recent).filter((e) => e.outlets.length >= need).sort((a, b) => b.outlets.length - a.outlets.length);
const missed = {};
for (const id of active) missed[id] = [];
for (const e of wide) for (const id of active) if (!e.outlets.includes(id)) missed[id].push({ title: e.articles[0].title, outlets: e.outlets.length, at: e.firstAt });
return { days, need, active: active.length, wide: wide.map((e) => ({ title: e.articles[0].title, outlets: e.outlets, at: e.firstAt })), missed };
}
// Front pages. Reads each outlet's home page (robots.txt allowing), finds the
// links that match reports this site has saved, and records their order.
const norm = (u) => { try { const x = new URL(u); return `${x.host.replace(/^www\./, '')}${x.pathname.replace(/\/+$/, '')}`; } catch { return ''; } };
async function placementSnapshot(outlets, saved, fetchPage) {
const byLink = {};
for (const f of Object.values(saved)) byLink[norm(f.link)] = f;
const res = await Promise.all(outlets.map(async (o) => {
if (!o.site) return [o.id, { ok: false, reason: 'no site' }];
const p = await fetchPage(o.site, { timeoutMs: 6000 });
if (!p.ok) return [o.id, { ok: false, reason: p.reason }];
const seen = new Set(); const top = [];
for (const m of p.html.matchAll(/<a\b[^>]*href=["']([^"'#]+)["']/gi)) {
let href; try { href = new URL(m[1], o.site).href; } catch { continue; }
const f = byLink[norm(href)];
if (!f || f.outletId !== o.id || seen.has(f.id)) continue;
seen.add(f.id); top.push({ id: f.id, rank: top.length + 1 });
if (top.length >= 5) break;
}
return [o.id, { ok: true, top }];
}));
return { at: new Date().toISOString(), outlets: Object.fromEntries(res) };
}
module.exports = { FEATURES_V, stillPaid, framePhrases, numberFrames, firstFigure, sponsorFrom, repetition, numberFraming, sponsors, coverage, placementSnapshot, GAIN, LOSS };