Source code

// The permanent archive. Nothing the site gathers is ever deleted from it.
//
// - archive/reports/YYYY-MM.json: every measurement of every report, by the
//   month the report was published. A report measured again with a newer word
//   list gets a second entry; the older measurement stays.
// - archive/scores/YYYY-MM-DD.json: the outlet score table, once a day.
// - archive/calibrations/<time>.json: every calibration run, in full.
// - archive/placement/YYYY-MM-DD.json: each day's front-page snapshots.
// - archive/score-trend.json: each day's outlet scores in one small file.
// - archive/index.json: the list of files and their sizes, for the archive page.
//
// The live site scores from the last 120 days; the archive keeps everything
// before and after. It stores counts, matched words with the few words around
// each one, headlines, links, a fingerprint of the text that was read and a
// link to the Internet Archive's copy. It does not store article text.

const R = require('./rules');
const { readJson, writeJson } = require('./store');

const INDEX = 'archive/index.json';
const TREND = 'archive/score-trend.json';
const memory = new Map(); // path -> object, guards against stale reads between quick writes

const monthOf = (iso) => (String(iso || '').match(/^\d{4}-\d{2}/) || ['undated'])[0];
const keyOf = (f) => `${f.id}@${f.lexiconFp || 'none'}`;

// Picks the more complete of two measurements made with the same word list.
function fuller(a, b) {
  if (!a) return b;
  if (!b) return a;
  if ((a.source === 'page') !== (b.source === 'page')) return a.source === 'page' ? a : b;
  return String(a.analyzedAt) >= String(b.analyzedAt) ? a : b;
}

async function readMerged(path) {
  const stored = await readJson(path, {});
  const mem = memory.get(path) || {};
  const out = { ...stored };
  for (const [k, v] of Object.entries(mem)) out[k] = fuller(out[k], v);
  return out;
}

async function touchIndex(entries) {
  const idx = { ...(await readJson(INDEX, {})), ...(memory.get(INDEX) || {}) };
  let changed = false;
  for (const [path, info] of Object.entries(entries)) {
    const prev = idx[path];
    if (!prev || prev.count !== info.count) { idx[path] = { ...info, updatedAt: new Date().toISOString() }; changed = true; }
  }
  if (changed) { memory.set(INDEX, idx); await writeJson(INDEX, idx); }
}

// Adds report measurements to the archive. Only months with a new or more
// complete record are written.
async function saveReports(records) {
  const byMonth = {};
  for (const f of records) {
    if (!f || !f.id || f.pending) continue;
    (byMonth[monthOf(f.published)] ||= []).push(f);
  }
  const written = {};
  for (const [month, list] of Object.entries(byMonth)) {
    const path = `archive/reports/${month}.json`;
    const file = await readMerged(path);
    let changed = false;
    for (const f of list) {
      const k = keyOf(f);
      const best = fuller(file[k], f);
      if (best !== file[k]) { file[k] = best; changed = true; }
    }
    if (changed) {
      memory.set(path, file);
      await writeJson(path, file);
      written[path] = { kind: 'reports', month, count: Object.keys(file).length };
    }
  }
  if (Object.keys(written).length) await touchIndex(written);
  return Object.keys(written);
}

// Saves the outlet score table once per day (the first update after midnight UTC).
async function saveDailyScores(outletScores, outlets) {
  const day = new Date().toISOString().slice(0, 10);
  const path = `archive/scores/${day}.json`;
  if (memory.has(path)) return false;
  const existing = await readJson(path, null);
  if (existing) { memory.set(path, existing); return false; }
  const snap = {
    day,
    takenAt: new Date().toISOString(),
    method: R.VERSION,
    ruleBookFingerprint: R.FINGERPRINT,
    outlets: outlets.map((o) => ({ id: o.id, name: o.name, feed: o.feed, ...(outletScores[o.id] || {}) })),
  };
  memory.set(path, snap);
  await writeJson(path, snap);
  await touchIndex({ [path]: { kind: 'scores', day, count: snap.outlets.length } });
  // A compact running record of each outlet's daily score, for trend views.
  const trend = memory.get(TREND) || (await readJson(TREND, {}));
  trend[day] = Object.fromEntries(snap.outlets.filter((o) => o.status === 'full' || o.status === 'provisional').map((o) => [o.id, { score: o.score, raw: o.raw, n: o.n, method: R.VERSION }]));
  memory.set(TREND, trend);
  await writeJson(TREND, trend);
  return true;
}

async function readTrend() { return memory.get(TREND) || (await readJson(TREND, {})); }

async function saveCalibration(result) {
  const at = new Date().toISOString();
  const path = `archive/calibrations/${at.replace(/[:.]/g, '-')}.json`;
  await writeJson(path, { at, method: R.VERSION, ...result });
  await touchIndex({ [path]: { kind: 'calibration', at, count: 1 } });
  return path;
}

// Front-page snapshots, all of a day's snapshots in one file.
async function savePlacement(snap) {
  const day = snap.at.slice(0, 10);
  const path = `archive/placement/${day}.json`;
  const list = memory.get(path) || (await readJson(path, []));
  if (!list.some((x) => x.at === snap.at)) list.push(snap);
  memory.set(path, list);
  await writeJson(path, list);
  await touchIndex({ [path]: { kind: 'placement', day, count: list.length } });
}

async function listArchive() {
  return { ...(await readJson(INDEX, {})), ...(memory.get(INDEX) || {}) };
}

module.exports = { saveReports, saveDailyScores, saveCalibration, savePlacement, listArchive, readTrend, readMerged, monthOf, keyOf };