fix: distinguish unreadable ledgers from missing evidence

CHANGELOG.md

@@ -5,6 +5,12 @@ Each release of Research-Kit, newest first. The version is `KIT_VERSION` in
 
 ## Unreleased
 
+- A leading ledger byte-order mark is tolerated while reading, without changing the
+  evidence file or its entry hashes. Tail repair leaves an intact BOM ledger alone
+  and preserves the prefix when dropping a torn append. Handoff gives unreadable lines the tail-repair
+  or restore-from-git remedy; secondary chain failures no longer imply failed travel
+  or lost entries while parsing is unresolved (#296).
+
 - ZIP archives declare UTF-8 names in both headers, so ordinary extractors preserve non-ASCII paths declared by the package manifest (#345).
 
 - Machine config saves preserve unpatched raw role and posture values, including typos

research-kit/README.md

@@ -568,7 +568,7 @@ node research-kit/bin/selftest.mjs            # all of it
 node research-kit/bin/selftest.mjs gate hook  # just these files
 ```
 
-1845 tests, offline, no key and no network. The runner **awaits** every test, so `ok` means
+1850 tests, offline, no key and no network. The runner **awaits** every test, so `ok` means
 the assertions settled (ADR-0021), and each test is raced against a watchdog
 (`RESEARCH_KIT_TEST_TIMEOUT`, default 60s).
 

research-kit/lib/corpus.mjs

@@ -495,7 +495,9 @@ export function readLedger(root) {
   const file = resolve(root, PATHS.ledger);
   const text = readText(file);
   if (text === null) return { present: false, entries: [], problems: [], lines: [] };
-  const lines = text.split(/\r?\n/);
+  // An editor's leading BOM is outside the entries the chain hashes. Ignore it while
+  // reading, as the JSON and Markdown readers do; never rewrite the evidence file.
+  const lines = text.replace(/^\uFEFF/, '').split(/\r?\n/);
   const entries = [];
   const problems = [];
   lines.forEach((line, idx) => {

research-kit/lib/handoff.mjs

@@ -319,14 +319,16 @@ export function verifyHandoff(root, { corpus = null } = {}) {
   const missingCaptures = [];
   const unledgered = [];
   const lineEndings = chain.lineEndings ?? [];
+  const ledgerUnparsed = chain.problems.filter((p) => p.rule === 'ledger-unparsed');
+  const parseConsequences = new Set();
 
   if (!snapshot.ledger.present) {
     findings.push({
       name: 'handoff-ledger-missing',
       severity: 'fail',
       detail: `${PATHS.ledger} is not in this checkout - the corpus arrived without its ledger`,
     });
-  } else if (!snapshot.ledger.entries.length) {
+  } else if (!snapshot.ledger.entries.length && !ledgerUnparsed.length) {
     findings.push({
       name: 'handoff-ledger-empty',
       severity: 'fail',
@@ -378,19 +380,24 @@ export function verifyHandoff(root, { corpus = null } = {}) {
 
   const structural = chain.problems.filter((p) => p.rule !== 'ledger-missing');
   for (const problem of structural) {
-    findings.push({
+    const finding = {
       name: 'handoff-chain-broken',
       severity: 'fail',
       // One line per capture, for the same reason the remedy is stated once: a
       // paragraph per capture buries the capture names.
       detail: `${problem.rule}${problem.kind ? `/${problem.kind}` : ''}: ${problem.detail}`,
       kind: problem.kind,
-    });
+    };
+    findings.push(finding);
+    // Missing sequence links cannot establish travel or loss while entries are unreadable.
+    // Restore parsing first; missing captures and altered bodies still have their own cause.
+    if (ledgerUnparsed.length && ['ledger-unparsed', 'seq', 'prev', 'entry-hash'].includes(problem.rule)) parseConsequences.add(finding);
   }
 
   // Present, empty, and captures on disk: lost here, not left behind.
   // Or entries missing for captures that are here - a torn last line of a longer ledger.
   const ledgerLost = snapshot.ledger.present
+    && !ledgerUnparsed.length
     && ((!snapshot.ledger.entries.length && snapshot.captures.entries.length > 0) || unledgered.length > 0);
   // A capture changed after its fetch is here, and so is its ledger entry: nothing failed to
   // travel, the bytes changed (alteredRemedy). A broken chain beside it still did.
@@ -399,6 +406,7 @@ export function verifyHandoff(root, { corpus = null } = {}) {
     .map((p) => ({ file: p.file, line: p.line }));
   const travelled = findings.filter((f) => (f.name !== 'handoff-chain-broken' || (f.kind !== 'line-endings' && f.kind !== 'modified'))
     && f.name !== 'handoff-capture-unledgered'
+    && !parseConsequences.has(f)
     && !(ledgerLost && f.name === 'handoff-ledger-empty'));
   const report = {
     root,
@@ -412,14 +420,19 @@ export function verifyHandoff(root, { corpus = null } = {}) {
     // changed after its fetch, or a ledger that lost entries here.
     didNotTravel: travelled.length > 0,
     ledgerLost,
+    ledgerUnparsed: ledgerUnparsed.map((p) => ({
+      line: p.line,
+      // Physical line numbers include blank lines, just as readLedger and tail repair do.
+      tail: p.line === snapshot.ledger.lines.findLastIndex((line) => line.trim()) + 1,
+    })),
     entries: snapshot.ledger.entries.length,
     rows: snapshot.evidence.length,
     // Not a finding: the corpus can arrive whole while the brief is unreviewed. The CLI
     // says so, because phase 2 starts from that file.
     brief: briefReview(snapshot),
     // No ledger entry, no evidence row, no capture: nothing was ever collected, so there is
     // nothing that could have failed to travel. The CLI says so on a collector.
-    nothingCollected: !snapshot.ledger.entries.length && !snapshot.evidence.length && !snapshot.captures.entries.length,
+    nothingCollected: !ledgerUnparsed.length && !snapshot.ledger.entries.length && !snapshot.evidence.length && !snapshot.captures.entries.length,
   };
   const remedy = handoffRemedy(report);
   if (remedy) report.findings.push({ name: 'handoff-remedy', severity: 'info', detail: remedy });
@@ -490,6 +503,11 @@ export function handoffRemedy(report) {
   const parts = [];
   if (report.didNotTravel) parts.push(HANDOFF_REMEDY);
   if (report.ledgerLost) parts.push(ledgerLostRemedy());
+  for (const problem of report.ledgerUnparsed ?? []) {
+    parts.push(problem.tail
+      ? `${PATHS.ledger} line ${problem.line}: the last line is torn (an interrupted append): ${kitCommand('doctor.mjs', '--fix-arity')} drops it. Run handoff again after repair.`
+      : `${PATHS.ledger} line ${problem.line}: a broken line inside the chain cannot be repaired here: restore research/raw/.fetches.jsonl from git, then run handoff again.`);
+  }
   // Whether this is inside a repository at all decides which remedies are even runnable.
   const { inside: isRepo, refused } = repositoryState(report.root ?? '.');
   if (report.altered?.length) parts.push(alteredRemedy(report.altered.map((e) => e.file), { isRepo, refused }));

research-kit/lib/provenance.mjs

@@ -544,7 +544,9 @@ export function repairLedgerTail(root) {
 
     const tail = lines[lines.length - 1];
     let torn = false;
-    try { JSON.parse(tail); } catch { torn = true; }
+    // A one-entry ledger can carry the same leading editor BOM readLedger tolerates.
+    // It is not a torn append; later-line BOMs remain invalid and the file is untouched.
+    try { JSON.parse(lines.length === 1 ? tail.replace(/^\uFEFF/, '') : tail); } catch { torn = true; }
     if (!torn) return { repaired: false, reason: 'tail-parses' };
 
     const head = lines.slice(0, -1).join('\n');

research-kit/test/corpus.test.mjs

@@ -4,14 +4,26 @@ import { spawnSync } from 'node:child_process';
 import { test, describe, assert, makePassingProject, makeProject, corrupt, fs, requireSymlink } from './harness.mjs';
 import { PATHS, HEADERS, resolve, writeText, readText, sha256File, isRegularFile } from '../lib/core.mjs';
 import {
-  readCorpus, readCaptures, parseTable, parseCapture, splitRow, escapeCell, tableRow,
+  readCorpus, readCaptures, readLedger, parseTable, parseCapture, splitRow, escapeCell, tableRow,
   repairRowArity, captureEntry, rememberCapture, cacheDecision, traceOf, captureOf,
   claimOf, sectionOf, appendRow, upsertRow, nextId, citedIds, stripReviewNotes, requestedUrl,
 } from '../lib/corpus.mjs';
 import { runPreflight } from '../lib/preflight.mjs';
 
 describe('corpus');
 
+test('a leading ledger byte-order mark preserves entries without rewriting the file', () => {
+  const dir = makePassingProject();
+  const file = resolve(dir, PATHS.ledger);
+  const before = readLedger(dir);
+  const saved = `\uFEFF${fs.readFileSync(file, 'utf8')}`;
+  fs.writeFileSync(file, saved);
+  const after = readLedger(dir);
+  assert.deepEqual(after.entries, before.entries);
+  assert.deepEqual(after.problems, []);
+  assert.equal(fs.readFileSync(file, 'utf8'), saved, 'reading must not edit the ledger');
+});
+
 test('splitRow honours escaped pipes; escapeCell round-trips through them', () => {
   assert.deepEqual(splitRow('| a | b\\|c | d |'), ['a', 'b|c', 'd']);
   assert.equal(escapeCell('b|c'), 'b\\|c');

research-kit/test/handoff.test.mjs

@@ -8,6 +8,8 @@ import { PATHS, resolve, writeText } from '../lib/core.mjs';
 import { verifyHandoff, handoffRemedy, HANDOFF_REMEDY, lineEndingRemedy, alteredRemedy, safePathspec, insideRepository, repositoryState, NAMES_FILE, PIN_LINES } from '../lib/handoff.mjs';
 import { readCorpus } from '../lib/corpus.mjs';
 import { runDoctor } from '../lib/doctor.mjs';
+import { appendFetch } from '../lib/provenance.mjs';
+import { runPreflight } from '../lib/preflight.mjs';
 
 describe('handoff');
 
@@ -32,6 +34,57 @@ test('a corpus that arrived whole passes', () => {
   assert.equal(report.remedy, '');
 });
 
+test('a leading ledger byte-order mark leaves the chain and handoff intact', () => {
+  const dir = makePassingProject();
+  corrupt(dir, PATHS.ledger, (text) => `\uFEFF${text}`);
+  const report = verifyHandoff(dir);
+  assert.equal(report.ok, true, JSON.stringify(report.findings));
+  assert.equal(report.didNotTravel, false);
+  assert.equal(report.ledgerLost, false);
+  assert.equal(runPreflight(dir).pass, true, 'the provenance gate must accept the same intact chain');
+});
+
+test('unparsed ledger lines get the local repair remedy instead of lost-entry or travel advice', () => {
+  const tail = makePassingProject();
+  fs.appendFileSync(resolve(tail, PATHS.ledger), '\n\n{"seq":2,"url":"unfinished');
+  let report = verifyHandoff(tail);
+  assert.equal(report.ok, false);
+  assert.equal(report.didNotTravel, false);
+  assert.equal(report.ledgerLost, false);
+  assert.match(report.remedy, /doctor\.mjs"? --fix-arity/);
+  assert.doesNotMatch(report.remedy, /Something did not travel|ledger lost entries/);
+
+  const middle = makePassingProject();
+  corrupt(middle, PATHS.ledger, (text) => `not JSON\n${text}`);
+  report = verifyHandoff(middle);
+  assert.equal(report.ok, false);
+  assert.equal(report.didNotTravel, false);
+  assert.equal(report.ledgerLost, false);
+  assert.match(report.remedy, /broken line inside the chain.*restore research\/raw\/\.fetches\.jsonl from git/);
+  assert.doesNotMatch(report.remedy, /Something did not travel|ledger lost entries|--fix-arity/);
+
+  const brokenEntry = makePassingProject();
+  appendFetch(brokenEntry, { op: 'fail', url: 'https://example.invalid/second', transport: 'http-keyless' });
+  corrupt(brokenEntry, PATHS.ledger, (text) => text.replace(/^[^\n]+/, 'unreadable first entry'));
+  report = verifyHandoff(brokenEntry);
+  assert.equal(report.ok, false);
+  assert.equal(report.didNotTravel, false, 'unparsed entries cause secondary seq/prev failures');
+  assert.equal(report.ledgerLost, false, 'a capture may be recorded in the unreadable entry');
+  assert.match(report.remedy, /restore research\/raw\/\.fetches\.jsonl from git/);
+  assert.doesNotMatch(report.remedy, /Something did not travel|ledger lost entries/);
+});
+
+test('an unreadable ledger does not hide a capture that really did not travel', () => {
+  const dir = makePassingProject();
+  const capture = readCorpus(dir).captures.entries[0];
+  fs.rmSync(resolve(dir, capture.file));
+  fs.appendFileSync(resolve(dir, PATHS.ledger), '{"seq":2');
+  const report = verifyHandoff(dir);
+  assert.equal(report.didNotTravel, true);
+  assert.match(report.remedy, /Something did not travel/);
+  assert.match(report.remedy, /--fix-arity/);
+});
+
 test('a missing ledger is named handoff-ledger-missing', () => {
   const dir = makePassingProject();
   fs.rmSync(resolve(dir, PATHS.ledger));

research-kit/test/provenance.test.mjs

@@ -18,6 +18,20 @@ test('happy path: the chain verifies and preflight passes', () => {
   assert.equal(runPreflight(dir).pass, true);
 });
 
+test('tail repair leaves an intact BOM ledger alone and preserves its prefix when dropping a torn append', () => {
+  const dir = makePassingProject();
+  const file = resolve(dir, PATHS.ledger);
+  const saved = `\uFEFF${fs.readFileSync(file, 'utf8')}`;
+  fs.writeFileSync(file, saved);
+  assert.deepEqual(repairLedgerTail(dir), { repaired: false, reason: 'tail-parses' });
+  assert.equal(fs.readFileSync(file, 'utf8'), saved);
+  assert.equal(verifyLedger(dir).ok, true);
+  fs.appendFileSync(file, '{"seq":2,"url":"unfinished');
+  assert.equal(repairLedgerTail(dir).repaired, true);
+  assert.equal(fs.readFileSync(file, 'utf8'), saved, 'repair must remove only the torn append');
+  assert.equal(verifyLedger(dir).ok, true);
+});
+
 test('a tampered capture fails body-unmodified', () => {
   const dir = makePassingProject();
   const capture = readCorpus(dir).captures.entries[0];