Source code

// Saved data, kept in the project's Vercel Blob stores. Each file is JSON.
// When no store is connected (local tests), files live in memory.
//
// Two stores:
// - the public store (BLOB_READ_WRITE_TOKEN) holds measurements, the archive
//   and other published data;
// - the private store (TEXT_READ_WRITE_TOKEN) holds the text of article pages
//   the site read with the outlet's permission. Nothing in it is published.
//   It is used only to measure a report again after a rule change and to
//   settle disputes. An outlet's copies are deleted on request.

const memory = new Map();
const textMemory = new Map();
const hasStore = () => Boolean(process.env.BLOB_READ_WRITE_TOKEN);
const hasTextStore = () => Boolean(process.env.TEXT_READ_WRITE_TOKEN) || !hasStore();
const publicAuth = () => ({ token: process.env.BLOB_READ_WRITE_TOKEN });
const textAuth = () => ({ token: process.env.TEXT_READ_WRITE_TOKEN });

async function readJson(path, fallback) {
  if (!hasStore()) return memory.has(path) ? JSON.parse(memory.get(path)) : fallback;
  try {
    const { get } = require('@vercel/blob');
    const r = await get(path, { access: 'public', useCache: false, ...publicAuth() });
    if (!r || r.statusCode !== 200) return fallback;
    const text = await new Response(r.stream).text();
    return JSON.parse(text);
  } catch (err) {
    if (err && /not.?found/i.test(err.name + err.message)) return fallback;
    console.error('read failed', path, err && err.message);
    return fallback;
  }
}

async function writeJson(path, value) {
  const body = JSON.stringify(value);
  if (!hasStore()) { memory.set(path, body); return; }
  const { put } = require('@vercel/blob');
  await put(path, body, { access: 'public', addRandomSuffix: false, allowOverwrite: true, contentType: 'application/json', cacheControlMaxAge: 60, ...publicAuth() });
}

// Article text, by outlet and report id: text/<outlet>/<report id>.json
const textPath = (outletId, id) => `text/${String(outletId).replace(/[^a-z0-9-]/gi, '')}/${String(id).replace(/[^a-z0-9]/gi, '')}.json`;

async function readText(outletId, id) {
  const path = textPath(outletId, id);
  if (!hasStore()) return textMemory.has(path) ? JSON.parse(textMemory.get(path)) : null;
  if (!process.env.TEXT_READ_WRITE_TOKEN) return null;
  try {
    const { get } = require('@vercel/blob');
    const r = await get(path, { access: 'private', ...textAuth() });
    if (!r || r.statusCode !== 200) return null;
    return JSON.parse(await new Response(r.stream).text());
  } catch (err) {
    if (!(err && /not.?found/i.test(err.name + err.message))) console.error('text read failed', path, err && err.message);
    return null;
  }
}

async function writeText(outletId, id, value) {
  const path = textPath(outletId, id);
  const body = JSON.stringify(value);
  if (!hasStore()) { textMemory.set(path, body); return true; }
  if (!process.env.TEXT_READ_WRITE_TOKEN) return false;
  const { put } = require('@vercel/blob');
  await put(path, body, { access: 'private', addRandomSuffix: false, allowOverwrite: true, contentType: 'application/json', ...textAuth() });
  return true;
}

// Every stored text for one outlet, for re-measuring and word mining.
async function listText(outletId, limit = 1000) {
  const prefix = outletId ? `text/${outletId}/` : 'text/';
  if (!hasStore()) return [...textMemory.keys()].filter((k) => k.startsWith(prefix)).slice(0, limit);
  if (!process.env.TEXT_READ_WRITE_TOKEN) return [];
  const { list } = require('@vercel/blob');
  const out = [];
  let cursor;
  do {
    const r = await list({ prefix, cursor, limit: 1000, ...textAuth() });
    out.push(...r.blobs.map((b) => b.pathname));
    cursor = r.hasMore ? r.cursor : null;
  } while (cursor && out.length < limit);
  return out.slice(0, limit);
}

// Deletes every stored text for one outlet (an outlet's request).
async function deleteTextFor(outletId) {
  const paths = await listText(outletId, 100000);
  if (!hasStore()) { for (const p of paths) textMemory.delete(p); return paths.length; }
  if (!paths.length) return 0;
  const { del } = require('@vercel/blob');
  for (let i = 0; i < paths.length; i += 500) await del(paths.slice(i, i + 500), textAuth());
  return paths.length;
}

async function readTextPath(path) {
  if (!hasStore()) return textMemory.has(path) ? JSON.parse(textMemory.get(path)) : null;
  const m = path.match(/^text\/([^/]+)\/([^/.]+)\.json$/);
  return m ? readText(m[1], m[2]) : null;
}

// Whether article text is being kept on the live site.
const textOn = () => Boolean(process.env.TEXT_READ_WRITE_TOKEN);

module.exports = { textOn, readJson, writeJson, hasStore, hasTextStore, readText, writeText, listText, deleteTextFor, readTextPath, textPath };