Source code

// Record checks (3.1.0): vote counts and bill numbers found in a report are
// looked up in the official record. Shown beside the score, never part of it.
// A count that is not found is reported as "does not match the record".
//
// Sources:
// - Senate roll-call votes: the Senate's own vote list for each session,
//   https://www.senate.gov/legislative/LIS/roll_call_lists/vote_menu_<congress>_<session>.xml
// - Bills: the Library of Congress API, https://api.congress.gov/v3/bill/...
//   (key from api.data.gov in CONGRESS_API_KEY; the shared DEMO_KEY otherwise,
//   which allows only a few lookups an hour).
// House roll-call counts are not checked yet.

const { readJson, writeJson } = require('./store');

const UA = 'TransparencyNewsBot/2.0 (+https://transparencynews.org/method)';
const CHECKS_PATH = 'records/checks.json';
const BILLS_PATH = 'records/bills.json';
const DAY = 86400000;
const MONTHS = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 };

function congressOf(date) {
  const y = date.getUTCFullYear();
  return { congress: Math.floor((y - 1789) / 2) + 1, session: y % 2 === 1 ? 1 : 2, year: y };
}

const tag = (x, t) => { const m = x.match(new RegExp(`<${t}>([\\s\\S]*?)</${t}>`)); return m ? m[1].replace(/<[^>]+>/g, '').trim() : ''; };

const senateMem = new Map();
async function senateVotes(date) {
  const { congress, session, year } = congressOf(date);
  const key = `${congress}_${session}`;
  const hit = senateMem.get(key);
  if (hit && Date.now() - hit.at < 3600 * 1000) return hit.votes;
  const url = `https://www.senate.gov/legislative/LIS/roll_call_lists/vote_menu_${key}.xml`;
  try {
    const r = await fetch(url, { headers: { 'user-agent': UA }, signal: AbortSignal.timeout(8000) });
    if (!r.ok) return hit ? hit.votes : null;
    const xml = await r.text();
    const votes = [...xml.matchAll(/<vote>([\s\S]*?)<\/vote>/g)].map(([, v]) => {
      const [d, mon] = tag(v, 'vote_date').split('-');
      return {
        number: +tag(v, 'vote_number'), date: Date.UTC(year, MONTHS[mon], +d), question: tag(v, 'question'), result: tag(v, 'result'),
        issue: tag(v, 'issue'), title: tag(v, 'title'), yeas: +tag(v, 'yeas'), nays: +tag(v, 'nays'),
        url: `https://www.senate.gov/legislative/LIS/roll_call_votes/vote${congress}${session}/vote_${congress}_${session}_${String(+tag(v, 'vote_number')).padStart(5, '0')}.htm`,
      };
    }).filter((v) => Number.isFinite(v.date) && v.number);
    senateMem.set(key, { at: Date.now(), votes });
    return votes;
  } catch { return hit ? hit.votes : null; }
}

// A Senate count is checked against roll calls from four days before the
// report's publication to one day after.
async function checkTally(t, published) {
  if (t.chamber !== 'Senate') return { ...t, result: 'not checked', why: t.chamber === 'House' ? 'House counts are not checked yet' : 'no chamber named in the sentence' };
  const at = new Date(published);
  const votes = await senateVotes(at);
  if (!votes) return { ...t, result: 'not checked', why: 'the Senate vote list could not be reached' };
  const near = votes.filter((v) => v.date >= at.getTime() - 4 * DAY && v.date <= at.getTime() + DAY);
  const match = near.find((v) => v.yeas === t.yea && v.nays === t.nay);
  if (match) return { ...t, result: 'matches', vote: match };
  return { ...t, result: 'does not match', why: `no Senate roll call with ${t.yea} to ${t.nay} between four days before and one day after publication (${near.length} roll calls in that span)` };
}

const BILL_TYPES = { hr: 'house-bill', s: 'senate-bill', hres: 'house-resolution', sres: 'senate-resolution', hjres: 'house-joint-resolution', sjres: 'senate-joint-resolution' };
async function billRecord(b, published, cache) {
  const { congress } = congressOf(new Date(published));
  const key = `${congress}-${b.type}-${b.number}`;
  const c = cache[key];
  if (c && Date.now() - c.at < DAY) return c;
  const apiKey = process.env.CONGRESS_API_KEY || 'DEMO_KEY';
  try {
    const r = await fetch(`https://api.congress.gov/v3/bill/${congress}/${b.type}/${b.number}?format=json&api_key=${encodeURIComponent(apiKey)}`, { headers: { 'user-agent': UA }, signal: AbortSignal.timeout(8000) });
    if (!r.ok) return c || { key, at: Date.now(), found: false, status: r.status };
    const j = await r.json();
    const bill = j.bill || {};
    const rec = {
      key, at: Date.now(), found: Boolean(bill.number), title: bill.title || '', latest: bill.latestAction ? { date: bill.latestAction.actionDate, text: bill.latestAction.text } : null,
      url: `https://www.congress.gov/bill/${congress}th-congress/${BILL_TYPES[b.type] || b.type}/${b.number}`,
    };
    cache[key] = rec;
    return rec;
  } catch { return c || { key, at: Date.now(), found: false }; }
}

// Checks the reports that name a vote count or a bill, up to a budget per
// update; results are saved and reused.
async function runChecks(reports, { budget = 12 } = {}) {
  const saved = await readJson(CHECKS_PATH, {});
  const bills = await readJson(BILLS_PATH, {});
  const todo = reports.filter((f) => ((f.tallies || []).length || (f.bills || []).length) && !(saved[f.id] && saved[f.id].fp === fpOf(f)));
  for (const f of todo.slice(0, budget)) {
    const tallies = [];
    for (const t of f.tallies || []) tallies.push(await checkTally(t, f.published));
    const recs = [];
    for (const b of (f.bills || []).slice(0, 3)) recs.push({ ...b, record: await billRecord(b, f.published, bills) });
    // Saved only when the records could be reached, so a failed lookup is tried again.
    const reached = !tallies.some((t) => /could not be reached/.test(t.why || '')) && recs.every((b) => b.record.found || b.record.status === 404);
    saved[f.id] = { fp: reached ? fpOf(f) : null, at: new Date().toISOString(), tallies, bills: recs };
  }
  if (todo.length) {
    const keep = new Set(reports.map((f) => f.id));
    for (const id of Object.keys(saved)) if (!keep.has(id)) delete saved[id];
    try { await writeJson(CHECKS_PATH, saved); await writeJson(BILLS_PATH, bills); } catch (err) { console.error('record checks save failed', err.message); }
  }
  return saved;
}
const fpOf = (f) => JSON.stringify([f.tallies || [], (f.bills || []).map((b) => b.type + b.number)]);

module.exports = { runChecks, checkTally, senateVotes, congressOf, CHECKS_PATH };