Source code

// Reads article pages so reports are scored on their full text. The reader
// follows each site's robots.txt and identifies itself. It never works around
// a refusal: no other reader name, proxy, cache or mirror. Where an outlet
// publishes its own plain-text edition (NPR's text.npr.org), that edition is
// read first, under its own robots.txt.

const UA = 'TransparencyNewsBot/2.0 (+https://transparencynews.org/method)';
const robotsCache = new Map();

async function robotsAllows(url) {
  const u = new URL(url);
  if (!robotsCache.has(u.host)) {
    robotsCache.set(u.host, (async () => {
      try {
        const r = await fetch(`${u.protocol}//${u.host}/robots.txt`, { headers: { 'user-agent': UA }, signal: AbortSignal.timeout(4000) });
        return r.ok ? parseRobots(await r.text()) : [];
      } catch { return []; }
    })());
  }
  const rules = await robotsCache.get(u.host);
  return pathAllowed(rules, u.pathname + u.search);
}

// The longest matching rule wins; on a tie, allow wins. "*" matches any
// characters and "$" marks the end of the path, as in Google's robots.txt rules.
function pathAllowed(rules, path) {
  let best = null;
  for (const rule of rules) {
    if (!ruleMatches(rule.path, path)) continue;
    if (!best || rule.path.length > best.path.length || (rule.path.length === best.path.length && rule.allow)) best = rule;
  }
  return !best || best.allow;
}

function ruleMatches(pattern, path) {
  const anchored = pattern.endsWith('$');
  const body = (anchored ? pattern.slice(0, -1) : pattern).split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*');
  return new RegExp(`^${body}${anchored ? '$' : ''}`).test(path);
}

// Keeps the rules for our reader by name, or for all readers ("*").
function parseRobots(txt) {
  const groups = [];
  let cur = null;
  for (const raw of txt.split(/\r?\n/)) {
    const line = raw.replace(/#.*/, '').trim();
    const m = line.match(/^(user-agent|allow|disallow)\s*:\s*(.*)$/i);
    if (!m) continue;
    const k = m[1].toLowerCase(); const v = m[2].trim();
    if (k === 'user-agent') {
      if (!cur || cur.rules.length) { cur = { agents: [], rules: [] }; groups.push(cur); }
      cur.agents.push(v.toLowerCase());
    } else if (cur && v) cur.rules.push({ allow: k === 'allow', path: v });
    else if (cur && k === 'disallow' && !v) cur.rules.push({ allow: true, path: '/' });
  }
  const mine = groups.filter((g) => g.agents.some((a) => a.includes('transparencynews')));
  return (mine.length ? mine : groups.filter((g) => g.agents.includes('*'))).flatMap((g) => g.rules);
}

// The outlet's own plain-text edition of a story, when it publishes one.
function textEdition(url) {
  try {
    const u = new URL(url);
    if (/(^|\.)npr\.org$/.test(u.hostname) && u.hostname !== 'text.npr.org') {
      const id = u.pathname.match(/\/(nx-s1-\d+)(?:\/|$)/) || u.pathname.match(/\/\d{4}\/\d{2}\/\d{2}\/(\d{6,})(?:\/|$)/);
      if (id) return `https://text.npr.org/${id[1]}`;
    }
  } catch { /* not a URL */ }
  return null;
}

async function fetchPage(url, opts = {}) {
  const alt = textEdition(url);
  if (alt) {
    const r = await fetchOne(alt, opts);
    if (r.ok) return { ...r, via: alt };
  }
  return fetchOne(url, opts);
}

async function fetchOne(url, { timeoutMs = 9000 } = {}) {
  if (!(await robotsAllows(url))) return { ok: false, reason: 'robots' };
  try {
    const r = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, redirect: 'follow', signal: AbortSignal.timeout(timeoutMs) });
    if (!r.ok) return { ok: false, reason: `http ${r.status}` };
    const html = await r.text();
    return { ok: true, html: html.slice(0, 1500000) };
  } catch (err) {
    return { ok: false, reason: err && err.name === 'TimeoutError' ? 'timeout' : 'unreachable' };
  }
}

module.exports = { textEdition, fetchPage, robotsAllows, parseRobots, pathAllowed, UA };