Skip to content

Commit 0553724

Browse files
merge: integrate verified fixes from #417 through #424
Preserve the eight reviewed source heads from #417 through #424 after all seven integration checks passed.
2 parents 54d899d + a332996 commit 0553724

23 files changed

Lines changed: 454 additions & 52 deletions

‎CHANGELOG.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,30 @@ Each release of Research-Kit, newest first. The version is `KIT_VERSION` in
55

66
## Unreleased
77

8+
- `collect-remote` refuses empty repository, topic, client-ref, prior and prefer
9+
values by name before creating an output directory or reading a token (#287).
10+
- Invalid `--snapshot-max-age-ms` values are usage errors with exit 2. Library callers
11+
receive `INPUT-SNAPSHOT-MAX-AGE` / `INCOMPLETE` rather than a false stale-snapshot
12+
verdict; finite non-negative allowances, including zero, retain their meaning (#343).
13+
- `decompose --limit` consistently requires a whole number of at least 1 before
14+
transport selection. `--max-scrapes 0` remains supported (#320).
15+
- Auto-collect Git failure diagnostics quote the first `fatal:` or `error:` cause,
16+
retaining the final-line fallback for output without either marker (#326).
17+
- A leading ledger byte-order mark is tolerated while reading, without changing the
18+
evidence file or its entry hashes. Tail repair leaves an intact BOM ledger alone
19+
and preserves the prefix when dropping a torn append. Handoff gives unreadable lines the tail-repair
20+
or restore-from-git remedy; secondary chain failures no longer imply failed travel
21+
or lost entries while parsing is unresolved (#296).
22+
- Finding extraction remembers the winning paragraph start and builds its excerpt
23+
forward only until truncation is established. Long unbroken captures avoid the
24+
quadratic array insertion while retaining the same finding and excerpt (#334).
25+
- Recipe loading refuses existing files without dimensions and blank dimension labels,
26+
naming the source and line. Decompose reports invalid recipe content with exit 2
27+
before writing the map (#356).
28+
- Oversized panel requests receive the intended 413 JSON refusal before the
29+
connection closes. Overflow discards retained body chunks and later data, while
30+
a complete JSON body at the existing byte limit remains accepted (#324).
31+
832
- ZIP archives declare UTF-8 names in both headers, so ordinary extractors preserve non-ASCII paths declared by the package manifest (#345).
933

1034
- Machine config saves preserve unpatched raw role and posture values, including typos

‎docs/ARCHITECTURE.md‎

Lines changed: 19 additions & 9 deletions
Large diffs are not rendered by default.

‎research-kit/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -568,7 +568,7 @@ node research-kit/bin/selftest.mjs # all of it
568568
node research-kit/bin/selftest.mjs gate hook # just these files
569569
```
570570

571-
1845 tests, offline, no key and no network. The runner **awaits** every test, so `ok` means
571+
1861 tests, offline, no key and no network. The runner **awaits** every test, so `ok` means
572572
the assertions settled (ADR-0021), and each test is raced against a watchdog
573573
(`RESEARCH_KIT_TEST_TIMEOUT`, default 60s).
574574

‎research-kit/bin/collect-remote.mjs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,9 @@ checkFlagValues(flags, {
150150
'search-transport': { choices: ['auto', 'serpapi', 'firecrawl-cli'] },
151151
timeout: { int: true, min: 1 },
152152
out: {}, workflow: {}, ref: {},
153+
repository: {}, topic: {}, 'client-ref': {}, prior: {}, prefer: {},
153154
}, { exit: EXIT.CANNOT_START });
154155

155-
if (flags['client-ref'] === true) {
156-
process.stderr.write('--client-ref needs a value\n');
157-
process.exit(EXIT.CANNOT_START);
158-
}
159156
const clientReference = checkClientRef(flags['client-ref']);
160157
if (!clientReference.ok) {
161158
process.stderr.write(`${redact(clientReference.detail)}\n`);

‎research-kit/bin/decompose.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ refuseUnknownFlags(flags, ['help', 'topic', 'dry-run', 'force', 'limit', 'max-sc
1818
// made 1, so `--max-scrapes` with no value quietly paid for one page.
1919
checkFlagValues(flags, {
2020
topic: 'value', recipe: 'value', transport: 'value', 'search-transport': 'value',
21-
limit: { int: true, min: 0 }, 'max-scrapes': { int: true, min: 0 },
21+
limit: { int: true, min: 1 }, 'max-scrapes': { int: true, min: 0 },
2222
});
2323
const root = process.cwd();
2424

@@ -131,7 +131,7 @@ try {
131131
process.stderr.write(`${why}\n`);
132132
process.exit(2);
133133
}
134-
if (err.code === 'UNKNOWN_RECIPE' || err.code === 'INVALID_BUDGET' || err.code === 'SEARCH_PROVIDER_NOT_READY') {
134+
if (err.code === 'UNKNOWN_RECIPE' || err.code === 'INVALID_RECIPE' || err.code === 'INVALID_BUDGET' || err.code === 'SEARCH_PROVIDER_NOT_READY') {
135135
process.stderr.write(`${err.message}\n`);
136136
process.exit(2);
137137
}

‎research-kit/bin/researcher-release.mjs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ function main(argv = process.argv.slice(2)) {
5454
if (parsed.error) { console.error(`${parsed.error}\n${usage()}`); return 2; }
5555
let result;
5656
if (parsed.command === 'validate') {
57+
const snapshotMaxAgeMs = parsed.snapshotmaxagems === undefined ? undefined : Number(parsed.snapshotmaxagems);
58+
if (parsed.snapshotmaxagems !== undefined
59+
&& (!parsed.snapshotmaxagems.trim() || !Number.isFinite(snapshotMaxAgeMs) || snapshotMaxAgeMs < 0)) {
60+
console.error(`--snapshot-max-age-ms must be a finite non-negative number\n${usage()}`);
61+
return 2;
62+
}
5763
if (!parsed.root || !parsed.package || !parsed.registry || !parsed.roles || !parsed.pointers) {
5864
console.error(usage());
5965
return 2;
@@ -78,7 +84,7 @@ function main(argv = process.argv.slice(2)) {
7884
headPath: parsed.head ? path.resolve(parsed.head) : null,
7985
genesisHash: parsed.genesishash ?? '0'.repeat(64),
8086
publicKeys,
81-
snapshotMaxAgeMs: parsed.snapshotmaxagems === undefined ? undefined : Number(parsed.snapshotmaxagems),
87+
snapshotMaxAgeMs,
8288
});
8389
} else if (parsed.command === 'conform') {
8490
if (!parsed.schema) { console.error(usage()); return 2; }

‎research-kit/lib/auto-collect.mjs‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,11 @@ const tail = (s, n = RUN_OUTPUT_BYTES) => {
114114
return bytes.length > n ? `[... earlier output truncated ...]\n${bytes.subarray(-n).toString('utf8')}` : text;
115115
};
116116
const lastLine = (s) => tail(String(s ?? '').trim().split(/\r?\n/).filter(Boolean).pop() ?? '', 1000);
117+
// Git puts the cause before multi-line advice; kit run summaries still use lastLine.
118+
const gitReason = (output) => {
119+
const cause = String(output ?? '').split(/\r?\n/).find((line) => /^(fatal|error):/.test(line));
120+
return cause === undefined ? lastLine(output) : tail(cause, 1000);
121+
};
117122
const posix = (p) => p.split(path.sep).join('/');
118123

119124
/** Legacy diagnostic parser. Collection accounting never trusts human-readable logs. */
@@ -182,7 +187,7 @@ export function createAutoCollect({
182187
const ahead = await git(['rev-list', '--count', '@{u}..HEAD'], top);
183188
if (ahead.truncated) return 'could not inspect local commits: Git output was truncated; deliver manually';
184189
const count = Number(ahead.output.trim());
185-
if (ahead.code !== 0) return `could not inspect local commits: ${lastLine(ahead.output)}`;
190+
if (ahead.code !== 0) return `could not inspect local commits: ${gitReason(ahead.output)}`;
186191
if (!/^\d+$/.test(ahead.output.trim()) || !Number.isSafeInteger(count)) {
187192
return 'could not inspect local commits: invalid Git ahead count; deliver manually';
188193
}
@@ -195,47 +200,47 @@ export function createAutoCollect({
195200
}
196201
const changed = await git(['diff-tree', '--no-commit-id', '--name-only', '-r', '-z', 'HEAD'], top);
197202
if (changed.truncated) return 'could not inspect pending commit paths: Git output was truncated; deliver manually';
198-
if (changed.code !== 0) return `could not inspect pending delivery commit: ${lastLine(changed.output)}`;
203+
if (changed.code !== 0) return `could not inspect pending delivery commit: ${gitReason(changed.output)}`;
199204
const allowed = changed.output.split('\0').filter(Boolean).every((file) => paths.some((base) => (
200205
file === base || file.startsWith(`${base.replace(/\/$/, '')}/`)
201206
)));
202207
if (!allowed) return 'the pending auto-collect commit contains paths outside this request; push it manually';
203208
const pushed = await git(['push'], top);
204-
return pushed.code === 0 ? 'pushed' : `push failed: ${lastLine(pushed.output)}`;
209+
return pushed.code === 0 ? 'pushed' : `push failed: ${gitReason(pushed.output)}`;
205210
}
206211
}
207212
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'research-kit-autocollect-index-'));
208213
const indexEnv = { ...env, GIT_INDEX_FILE: path.join(scratch, 'index') };
209214
const isolatedGit = (args) => exec('git', args, { cwd: top, env: indexEnv });
210215
try {
211216
const initialized = await isolatedGit(['read-tree', 'HEAD']);
212-
if (initialized.code !== 0) return `could not initialize isolated git index: ${lastLine(initialized.output)}`;
217+
if (initialized.code !== 0) return `could not initialize isolated git index: ${gitReason(initialized.output)}`;
213218
for (const p of paths) {
214219
const r = await isolatedGit(['add', '--', p]);
215-
if (r.code !== 0) return `git add ${p} failed: ${lastLine(r.output)}`;
220+
if (r.code !== 0) return `git add ${p} failed: ${gitReason(r.output)}`;
216221
}
217222
for (const p of paths) {
218223
if (!(p === 'research' || p.endsWith('/research'))) continue;
219224
const ledger = path.posix.join(p.slice(0, -'research'.length) || '.', LEDGER);
220225
if (fs.existsSync(path.join(top, ledger))) {
221226
const r = await isolatedGit(['add', '-f', '--', ledger]);
222-
if (r.code !== 0) return `git add -f ${ledger} failed: ${lastLine(r.output)}`;
227+
if (r.code !== 0) return `git add -f ${ledger} failed: ${gitReason(r.output)}`;
223228
}
224229
}
225230
const staged = await isolatedGit(['diff', '--cached', '--quiet']);
226-
if (staged.code !== 0 && staged.code !== 1) return `could not inspect isolated git index: ${lastLine(staged.output)}`;
231+
if (staged.code !== 0 && staged.code !== 1) return `could not inspect isolated git index: ${gitReason(staged.output)}`;
227232
const hasChanges = staged.code !== 0;
228233
if (hasChanges) {
229234
const c = await isolatedGit(['commit', '-m', report.subject, '-m', report.body]);
230-
if (c.code !== 0) return `commit refused: ${lastLine(c.output)}`;
235+
if (c.code !== 0) return `commit refused: ${gitReason(c.output)}`;
231236
const reset = await git(['reset', '--quiet', 'HEAD', '--', ...paths], top);
232-
if (reset.code !== 0) return `committed; could not refresh staged paths: ${lastLine(reset.output)}`;
237+
if (reset.code !== 0) return `committed; could not refresh staged paths: ${gitReason(reset.output)}`;
233238
}
234239
const upstream = await git(['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], top);
235240
if (upstream.truncated) return `${hasChanges ? 'committed; ' : ''}could not inspect upstream: Git output was truncated; deliver manually`;
236241
if (upstream.code !== 0) return hasChanges ? 'committed; no upstream to push to' : 'no upstream to push to';
237242
const pushed = await git(['push'], top);
238-
if (pushed.code !== 0) return hasChanges ? `committed; push failed: ${lastLine(pushed.output)}` : `push failed: ${lastLine(pushed.output)}`;
243+
if (pushed.code !== 0) return hasChanges ? `committed; push failed: ${gitReason(pushed.output)}` : `push failed: ${gitReason(pushed.output)}`;
239244
return hasChanges ? 'committed and pushed' : 'pushed';
240245
} finally {
241246
fs.rmSync(scratch, { recursive: true, force: true });
@@ -290,7 +295,7 @@ export function createAutoCollect({
290295
if (!inside(canonical(top), physical)) return { problem: 'the approved topics folder resolves outside the repository' };
291296
const gitDir = await git(['rev-parse', '--absolute-git-dir'], top);
292297
if (gitDir.truncated) return { problem: 'could not inspect Git metadata: Git output was truncated; inspect the target manually' };
293-
if (gitDir.code !== 0) return { problem: `could not inspect Git metadata: ${lastLine(gitDir.output)}` };
298+
if (gitDir.code !== 0) return { problem: `could not inspect Git metadata: ${gitReason(gitDir.output)}` };
294299
if (gitDir.code === 0 && inside(canonical(gitDir.output.trim()), physical)) {
295300
return { problem: 'new topics cannot be created in Git metadata' };
296301
}
@@ -327,12 +332,12 @@ export function createAutoCollect({
327332
const pathspecs = [...paths, `:(exclude)${excludedPath}`];
328333
const status = await git(['status', '--porcelain=v1', '-z', '--untracked-files=all', '--', ...pathspecs], top);
329334
if (status.truncated) throw new Error('could not inspect pending delivery: Git output was truncated; deliver manually');
330-
if (status.code !== 0) throw new Error(`could not inspect pending delivery: ${lastLine(status.output)}`);
335+
if (status.code !== 0) throw new Error(`could not inspect pending delivery: ${gitReason(status.output)}`);
331336
const hash = crypto.createHash('sha256');
332337
hash.update(status.output);
333338
const diff = await git(['diff', '--binary', 'HEAD', '--', ...pathspecs], top);
334339
if (diff.truncated) throw new Error('could not fingerprint pending delivery: Git output was truncated; deliver manually');
335-
if (diff.code !== 0) throw new Error(`could not fingerprint pending delivery: ${lastLine(diff.output)}`);
340+
if (diff.code !== 0) throw new Error(`could not fingerprint pending delivery: ${gitReason(diff.output)}`);
336341
hash.update(diff.output);
337342
for (const record of status.output.split('\0')) {
338343
if (!record.startsWith('?? ')) continue;
@@ -563,7 +568,7 @@ export function createAutoCollect({
563568
const top = await repoTop(dir);
564569
if (!top) { note(`${dir} is not in a git repository; requests travel by git`); return status(); }
565570
const pulled = await git(['pull', '--ff-only'], top);
566-
if (pulled.code !== 0) note(`git pull did not run cleanly (${lastLine(pulled.output)}); collection blocked until synchronization succeeds`);
571+
if (pulled.code !== 0) note(`git pull did not run cleanly (${gitReason(pulled.output)}); collection blocked until synchronization succeeds`);
567572
const requests = pulled.code === 0 ? listRequests(dir) : [];
568573

569574
for (const item of requests) {

‎research-kit/lib/corpus.mjs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -495,7 +495,9 @@ export function readLedger(root) {
495495
const file = resolve(root, PATHS.ledger);
496496
const text = readText(file);
497497
if (text === null) return { present: false, entries: [], problems: [], lines: [] };
498-
const lines = text.split(/\r?\n/);
498+
// An editor's leading BOM is outside the entries the chain hashes. Ignore it while
499+
// reading, as the JSON and Markdown readers do; never rewrite the evidence file.
500+
const lines = text.replace(/^\uFEFF/, '').split(/\r?\n/);
499501
const entries = [];
500502
const problems = [];
501503
lines.forEach((line, idx) => {

‎research-kit/lib/decompose.mjs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,19 @@ export function parseRecipe(text) {
2828
let name = '';
2929
if (match) {
3030
let inList = false;
31-
for (const line of match[1].split(/\r?\n/)) {
31+
for (const [offset, line] of match[1].split(/\r?\n/).entries()) {
3232
const named = line.match(/^name:\s*(.*)$/);
3333
if (named) { name = named[1].trim(); continue; }
3434
if (/^dimensions:\s*$/.test(line)) { inList = true; continue; }
3535
if (inList) {
3636
const item = line.match(/^\s*-\s+(.*)$/);
3737
if (!item) { if (line.trim()) inList = false; continue; }
3838
const [label, why] = item[1].split(/\s*\|\s*/);
39+
if (!label.trim()) {
40+
const err = new Error(`line ${offset + 2}: the dimension label is blank`);
41+
err.code = 'INVALID_RECIPE';
42+
throw err;
43+
}
3944
dimensions.push({ name: label.trim(), why: (why ?? '').trim() });
4045
}
4146
}
@@ -54,7 +59,17 @@ export function loadRecipe(nameOrPath) {
5459
err.code = 'UNKNOWN_RECIPE';
5560
throw err;
5661
}
57-
return { ...parseRecipe(text), file: direct };
62+
let parsed;
63+
try { parsed = parseRecipe(text); } catch (err) {
64+
if (err.code === 'INVALID_RECIPE') err.message = `recipe "${direct}" ${err.message}`;
65+
throw err;
66+
}
67+
if (!parsed.dimensions.length) {
68+
const err = new Error(`recipe "${direct}" holds no dimensions - use a front-matter dimensions list of "- label | why" lines; see ${RECIPE_DIR}`);
69+
err.code = 'INVALID_RECIPE';
70+
throw err;
71+
}
72+
return { ...parsed, file: direct };
5873
}
5974

6075
/** Where the facts probably live: the official hosts a search keeps pointing at. */

‎research-kit/lib/finding.mjs‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,9 @@ export function findingWithContext(markdown, fallback = '') {
331331

332332
for (let i = 0; i < lines.length; i += 1) {
333333
const raw = lines[i];
334+
// Excerpts use physical paragraph boundaries, including ones inside skipped fences.
335+
// Remember the start while scanning so recovery never inserts each line at the front.
336+
if (!raw.trim() || /^\s{0,3}#{1,6}\s/.test(raw)) blockStart = i + 1;
334337
if (/^\s*```/.test(raw)) { inFence = !inFence; continue; }
335338
if (inFence) continue;
336339

@@ -363,15 +366,15 @@ export function findingWithContext(markdown, fallback = '') {
363366
// An FAQ question is usually plain text, not a heading, so the heading rules cannot
364367
// see it. Remember whether the previous non-empty line asked something.
365368
const answersQuestion = lastLine.endsWith('?');
366-
if (line) { if (!block.length) blockStart = i; block.push(line); lastLine = line; }
369+
if (line) { block.push(line); lastLine = line; }
367370
if (isNoise(line)) continue;
368371
for (const sentence of sentences(line)) {
369372
const text = sentence.trim();
370373
if (text.length < 25) continue;
371374
if (isNoise(text)) continue;
372375
// The excerpt is resolved after the paragraph closes, so a sentence carries the
373376
// whole paragraph it came from rather than just itself.
374-
candidates.push({ text, fromTable: false, heading, headings: stack.filter(Boolean).join(' / '), answersQuestion, lineIndex: i });
377+
candidates.push({ text, fromTable: false, heading, headings: stack.filter(Boolean).join(' / '), answersQuestion, lineIndex: i, blockStart });
375378
}
376379
}
377380
flushBlock();
@@ -396,10 +399,13 @@ export function findingWithContext(markdown, fallback = '') {
396399
// Recover the paragraph the winner sat in, bounded. A table row already has one.
397400
let excerpt = best.excerpt ?? '';
398401
if (!excerpt && best.lineIndex !== undefined) {
399-
const para = [];
400-
for (let i = best.lineIndex; i >= 0 && lines[i]?.trim() && !/^\s{0,3}#{1,6}\s/.test(lines[i]); i -= 1) para.unshift(stripMarkdown(lines[i]));
401-
for (let i = best.lineIndex + 1; i < lines.length && lines[i]?.trim() && !/^\s{0,3}#{1,6}\s/.test(lines[i]); i += 1) para.push(stripMarkdown(lines[i]));
402-
excerpt = para.filter(Boolean).join(' ').trim();
402+
for (let i = best.blockStart; i < lines.length && lines[i]?.trim() && !/^\s{0,3}#{1,6}\s/.test(lines[i]); i += 1) {
403+
const line = stripMarkdown(lines[i]);
404+
if (!line) continue;
405+
excerpt += `${excerpt ? ' ' : ''}${line}`;
406+
// One character beyond the cap proves truncation, including an exact-cap prefix.
407+
if (excerpt.length > EXCERPT_CHARS) break;
408+
}
403409
}
404410
if (excerpt.length > EXCERPT_CHARS) excerpt = `${excerpt.slice(0, EXCERPT_CHARS).trimEnd()}…`;
405411

0 commit comments

Comments
 (0)