Source code

// Turns an article page into plain paragraphs, and separates the outlet's own
// words from the words it quotes.

const { decode } = require('./rss');

// Pulls readable paragraphs from an article page. Prefers the <article> block
// when the page has one, drops scripts, navigation and very short lines.
function extractParagraphs(html) {
  if (!html) return [];
  let body = html.replace(/<(script|style|noscript|svg|nav|footer|header|aside|form|figure|figcaption)[\s\S]*?<\/\1>/gi, ' ');
  const art = body.match(/<article[\s>][\s\S]*?<\/article>/i);
  if (art && art[0].length > 2000) body = art[0];
  const paras = [];
  const re = /<p[\s>][\s\S]*?<\/p>/gi;
  let m;
  while ((m = re.exec(body))) {
    const t = decode(m[0]);
    if (t.split(/\s+/).length >= 8 && !/^(advertisement|subscribe|sign up|copyright|©|read more|related:|click here)/i.test(t)) paras.push(t);
  }
  const out = dedupe(paras);
  // Fallback (3.1.0): many sites embed the full story in the page's
  // schema.org data (articleBody), which is part of the page as served.
  if (countWords(out.join(' ')) < 150) {
    const fromData = articleBody(html);
    if (fromData.length && countWords(fromData.join(' ')) > countWords(out.join(' '))) return fromData;
  }
  return out;
}

function articleBody(html) {
  let best = '';
  for (const m of String(html).matchAll(/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) {
    try {
      const walk = (x) => {
        if (!x || typeof x !== 'object') return;
        if (Array.isArray(x)) { x.forEach(walk); return; }
        if (typeof x.articleBody === 'string' && x.articleBody.length > best.length) best = x.articleBody;
        for (const v of Object.values(x)) if (v && typeof v === 'object') walk(v);
      };
      walk(JSON.parse(m[1]));
    } catch { /* not valid JSON: skipped */ }
  }
  if (!best) return [];
  const text = decode(best);
  const parts = text.split(/\n{1,}|(?<=[.?!”"])\s{2,}/).map((t) => t.trim()).filter((t) => t.split(/\s+/).length >= 8);
  return dedupe(parts.length ? parts : [text]);
}

function dedupe(list) {
  const seen = new Set();
  return list.filter((p) => (seen.has(p) ? false : (seen.add(p), true)));
}

function countWords(text) {
  const m = String(text || '').match(/[A-Za-z0-9][A-Za-z0-9'’\-]*/g);
  return m ? m.length : 0;
}

const SAID = /\b(said|says|told|asked|wrote|added|stated|declared|according to|tweeted|posted|argued|called it|called the|described|testified|announced)\b/i;

// Removes quoted speech so only the outlet's own words are scored. A short
// quoted phrase (one to three words) with no speaker in the same sentence, within 60 characters, is a
// scare quote (3.1.0): the reporter chose to put it there, so its words stay
// in the outlet's own text and count in full. It is also listed separately.
function ownVoice(text) {
  const scare = [];
  const quotes = [];
  let quotedWords = 0;
  const handle = (match, inner, offset, whole) => {
    const n = countWords(inner);
    if (n >= 1 && n <= 3) {
      // A speaker counts only within the same sentence and 60 characters.
      let left = whole.slice(Math.max(0, offset - 60), offset);
      let right = whole.slice(offset + match.length, offset + match.length + 60);
      left = left.slice(Math.max(left.lastIndexOf('. '), left.lastIndexOf('? '), left.lastIndexOf('! ')) + 1);
      const end = right.search(/[.?!](\s|$)/);
      if (end >= 0) right = right.slice(0, end + 1);
      const near = `${left}${match}${right}`;
      if (!SAID.test(near)) { scare.push(inner.trim()); return ` ${inner.trim()} `; }
    }
    quotedWords += n;
    quotes.push(inner.trim());
    return ' ';
  };
  let out = String(text || '');
  out = out.replace(/"([^"]{1,600})"/g, handle);
  out = out.replace(/“([^”]{1,600})”/g, handle);
  out = out.replace(/‘([^’]{1,300})’(?![a-z])/gi, handle);
  out = out.replace(/(?<![A-Za-z])'([^']{1,300})'(?![A-Za-z])/g, handle);
  return { text: out.replace(/\s+/g, ' ').trim(), scare, quotes, quotedWords };
}

function sentences(text) {
  return String(text || '').split(/(?<=[.?!])\s+(?=[A-Z"“])/).filter((s) => countWords(s) >= 4);
}

module.exports = { articleBody, extractParagraphs, countWords, ownVoice, sentences };