Source code

// Reads RSS 2.0 and Atom feeds into a common article shape.
// No outside libraries: feeds are simple enough to parse with careful regexes,
// and fewer dependencies means fewer things to break.

const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', '#39': "'", rsquo: '’', lsquo: '‘', rdquo: '”', ldquo: '“', mdash: '—', ndash: '–', hellip: '…' };

function decode(s) {
  if (!s) return '';
  return s
    .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16)))
    .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
    .replace(/&([a-z0-9#]+);/gi, (m, n) => (n.toLowerCase() in ENTITIES ? ENTITIES[n.toLowerCase()] : m))
    .replace(/<[^>]+>/g, ' ') // tags that were entity-encoded inside the field
    .replace(/\s+/g, ' ')
    .trim();
}

function tag(block, name) {
  const re = new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, 'i');
  const m = block.match(re);
  return m ? m[1] : '';
}

function attr(block, name, attribute) {
  const re = new RegExp(`<${name}\\s[^>]*${attribute}=["']([^"']+)["'][^>]*/?>`, 'i');
  const m = block.match(re);
  return m ? m[1] : '';
}

function categories(block) {
  return (block.match(/<category[^>]*>[\s\S]*?<\/category>/gi) || []).map((c) => decode(c.replace(/^<category[^>]*>|<\/category>$/gi, ''))).filter(Boolean).slice(0, 12);
}

function parseDate(s) {
  const t = Date.parse(decode(s));
  return Number.isFinite(t) ? new Date(t).toISOString() : null;
}

// Returns [{ title, link, summary, published }]
function parseFeed(xml) {
  const out = [];
  const items = xml.match(/<item[\s>][\s\S]*?<\/item>/gi);
  if (items) {
    for (const it of items) {
      out.push({
        title: decode(tag(it, 'title')),
        link: decode(tag(it, 'link')) || decode(tag(it, 'guid')),
        summary: decode(tag(it, 'description')),
        published: parseDate(tag(it, 'pubDate') || tag(it, 'dc:date')),
        categories: categories(it),
      });
    }
    return out.filter((a) => a.title && a.link);
  }
  const entries = xml.match(/<entry[\s>][\s\S]*?<\/entry>/gi) || [];
  for (const e of entries) {
    out.push({
      title: decode(tag(e, 'title')),
      link: attr(e, 'link', 'href') || decode(tag(e, 'link')),
      summary: decode(tag(e, 'summary') || tag(e, 'content')),
      published: parseDate(tag(e, 'published') || tag(e, 'updated')),
      categories: (e.match(/<category\s[^>]*term=["']([^"']+)["']/gi) || []).map((c) => c.replace(/^.*term=["']|["']$/g, '')),
    });
  }
  return out.filter((a) => a.title && a.link);
}

async function fetchOutlet(outlet, { timeoutMs = 8000 } = {}) {
  const started = Date.now();
  try {
    const res = await fetch(outlet.feed, {
      headers: { 'user-agent': 'TransparencyNews/0.1 (+https://transparencynews.org/about)', accept: 'application/rss+xml, application/atom+xml, application/xml, text/xml' },
      signal: AbortSignal.timeout(timeoutMs),
      redirect: 'follow',
    });
    if (!res.ok) return { outlet, ok: false, error: `The feed answered with error ${res.status}`, articles: [], ms: Date.now() - started };
    const xml = await res.text();
    const articles = parseFeed(xml).map((a) => ({ ...a, outletId: outlet.id }));
    return { outlet, ok: true, articles, ms: Date.now() - started };
  } catch (err) {
    const reason = err && err.name === 'TimeoutError' ? 'The feed did not answer in time' : 'The feed could not be reached';
    return { outlet, ok: false, error: reason, articles: [], ms: Date.now() - started };
  }
}

async function fetchAll(outlets, opts) {
  return Promise.all(outlets.map((o) => fetchOutlet(o, opts)));
}

module.exports = { parseFeed, decode, fetchOutlet, fetchAll };