From 4b9df2a83663548b07a013075c8b81136216586f Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Wed, 2 Sep 2026 22:55:23 -0700 Subject: [PATCH 01/30] Read the per-file FDR sidecars in bounded chunks instead of whole * Replaced File.ReadAllBytes in FdrScoresSidecar.TryRead and TryReadOverlay with TryWalkRecords, a shared header validation plus a 2,048-record buffered walk of the body. Same bytes, same order, same result, no format change * Removes a 106 MB large-object allocation PER FILE from Stage 7's pool rebuild, which reads every run's pre-compaction 1st-pass sidecar from inside a Parallel.For - 47 GB of LOH arrays at 446 runs to land the ~648 K records per run that survive compaction. Stage 7's band is Server-GC retained COMMITTED memory rather than live data, so burst allocation of that shape inflates it directly * Added TestFdrScoresSidecarChunkBoundaries over counts 0, 1, 1023, 1024, 1025, 2047, 2048, 2049, 4096 and 4103. Every other sidecar test writes a handful of records and would pass against a reader that dropped or misaligned everything past the first buffer NOT YET GATED: regression.ps1 cannot run while the 446-file measurement run holds the box, so this has build + 603 unit tests + zero-warning inspection only. Run -Dataset Stellar then -Dataset All before merging. See TODO-20260901_osprey_stage5_reload_materialization.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.IO/FdrScoresSidecar.cs | 232 ++++++++++-------- pwiz_tools/Osprey/Osprey.Test/IOTest.cs | 70 ++++++ 2 files changed, 195 insertions(+), 107 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs index 7273061b60..b0744f2ef8 100644 --- a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs @@ -138,6 +138,14 @@ public static class FdrScoresSidecar public const int HeaderLength = 32; public const int RecordLength = 28; + /// + /// Records per buffered body read in . 2,048 x 28 B = + /// 57,344 B, comfortably under the 85,000-byte large-object threshold, so a reader + /// walks a 106 MB sidecar through one Gen0 buffer instead of allocating the whole + /// file on the LOH. + /// + private const int RECORDS_PER_CHUNK = 2048; + /// /// Pass identifier embedded in the header. Mirrors the Rust pass /// byte semantics: 1 = first-pass Percolator, 2 = second-pass @@ -600,84 +608,38 @@ public static bool TryRead(string path, IList entries, Pass expectedPa if (path == null) throw new ArgumentNullException(nameof(path)); if (entries == null) throw new ArgumentNullException(nameof(entries)); - byte[] data; - try - { - data = File.ReadAllBytes(path); - } - // NOT a bare catch: an OutOfMemoryException here is reported as a MISSING - // sidecar, and a missing 1st-pass sidecar leaves those entries at Score 0.0. - // The decoy side is not q-gated, so the zeros then compete in the picked- - // protein null and the run exits 0 with corrupted protein numbers. Let it - // propagate and kill the run instead (#4615 review). - catch (Exception ex) when (!(ex is OutOfMemoryException)) - { - return false; - } - - if (data.Length < HeaderLength) - return false; - for (int i = 0; i < Magic.Length; i++) - { - if (data[i] != Magic[i]) - return false; - } - byte version = data[8]; - if (version != FormatVersion) - return false; - // Reject mismatched pass bytes so a 2nd-pass sidecar can never - // be silently loaded into 1st-pass stubs (or vice versa) — the - // q-values would scramble without any visible error. - byte passByte = data[9]; - if (passByte != (byte)expectedPass) - return false; - // bytes 10..16 reserved, ignored - ulong headerCount = BitConverter.ToUInt64(data, 16); - // Reject sidecars whose declared count exceeds physical - // record capacity. (headerCount can validly be < entries - // count — see comment above on pre-gap-fill / post- - // compaction sidecars.) Use checked arithmetic so a - // corrupt or malicious sidecar with a huge headerCount - // is rejected loudly instead of wrapping int silently. - if (!TryComputeExpectedLen(headerCount, out int expectedLen)) - return false; - if (data.Length != expectedLen) - return false; - // Build lookup so position-skewed entries align by entry_id. // Single-file degenerates to a 1:1 map (no perf cost vs the - // old positional walk). + // old positional walk). Built before the file is opened rather than after the + // header validates: on the happy path it is the same work, and the only path it + // is wasted on is one that returns false and fails the run anyway. var byEntryId = new Dictionary(entries.Count); for (int i = 0; i < entries.Count; i++) byEntryId[entries[i].EntryId] = i; - for (int rec = 0; rec < (int)headerCount; rec++) + return TryWalkRecords(path, expectedPass, (chunk, off) => { - int off = HeaderLength + rec * RecordLength; - uint recordEntryId = BitConverter.ToUInt32(data, off + 0); + uint recordEntryId = BitConverter.ToUInt32(chunk, off + 0); if (!byEntryId.TryGetValue(recordEntryId, out int entryIdx)) { // A caller that filtered its stub list says so by supplying the // predicate that did the filtering; a record it dropped on purpose // is expected to have no entry here. - if (expectedAbsent != null && expectedAbsent(recordEntryId)) - continue; - // Sidecar carries an entry the caller's stub list - // doesn't contain. The caller is expected to pass - // a SUPERSET of the sidecar's entries (the post- - // rescore parquet for the 1st-pass sidecar, for - // example) — a record that fails to find its - // entry_id signals the sidecar was written from a - // different parquet (or from a different binary - // version with different entry_id assignment). That - // is corruption, not the gap-fill or post-compaction - // case we tolerate, and must be rejected. - return false; + // + // Otherwise the sidecar carries an entry the caller's stub list + // doesn't contain. The caller is expected to pass a SUPERSET of the + // sidecar's entries (the post-rescore parquet for the 1st-pass + // sidecar, for example) — a record that fails to find its entry_id + // signals the sidecar was written from a different parquet (or from a + // different binary version with different entry_id assignment). That + // is corruption, not the gap-fill or post-compaction case we tolerate, + // and must be rejected. + return expectedAbsent != null && expectedAbsent(recordEntryId); } var e = entries[entryIdx]; - e.Score = BitConverter.ToDouble(data, off + 4); - e.RunPrecursorQvalue = BitConverter.ToDouble(data, off + 12); - e.RunPeptideQvalue = BitConverter.ToDouble(data, off + 20); + e.Score = BitConverter.ToDouble(chunk, off + 4); + e.RunPrecursorQvalue = BitConverter.ToDouble(chunk, off + 12); + e.RunPeptideQvalue = BitConverter.ToDouble(chunk, off + 20); // The EXPERIMENT-scope half, applied HERE so it reaches exactly the entries this // sidecar has a record for and no others (format v5, issue #4486). // @@ -701,8 +663,8 @@ public static bool TryRead(string path, IList entries, Pass expectedPa e.ExperimentProteinQvalue = exp.ExperimentProteinQvalue; e.ExperimentAggregateScore = exp.ExperimentAggregateScore; } - } - return true; + return true; + }); } /// @@ -721,55 +683,20 @@ public static bool TryReadOverlay(string path, if (path == null) throw new ArgumentNullException(nameof(path)); if (entriesByEntryId == null) throw new ArgumentNullException(nameof(entriesByEntryId)); - byte[] data; - try - { - data = File.ReadAllBytes(path); - } - // NOT a bare catch: an OutOfMemoryException here is reported as a MISSING - // sidecar, and a missing 1st-pass sidecar leaves those entries at Score 0.0. - // The decoy side is not q-gated, so the zeros then compete in the picked- - // protein null and the run exits 0 with corrupted protein numbers. Let it - // propagate and kill the run instead (#4615 review). - catch (Exception ex) when (!(ex is OutOfMemoryException)) - { - return false; - } - - if (data.Length < HeaderLength) - return false; - for (int i = 0; i < Magic.Length; i++) - { - if (data[i] != Magic[i]) - return false; - } - byte version = data[8]; - if (version != FormatVersion) - return false; - byte passByte = data[9]; - if (passByte != (byte)expectedPass) - return false; - ulong headerCount = BitConverter.ToUInt64(data, 16); - if (!TryComputeExpectedLen(headerCount, out int expectedLen)) - return false; - if (data.Length != expectedLen) - return false; - - for (int rec = 0; rec < (int)headerCount; rec++) + return TryWalkRecords(path, expectedPass, (chunk, off) => { - int off = HeaderLength + rec * RecordLength; - uint recordEntryId = BitConverter.ToUInt32(data, off + 0); + uint recordEntryId = BitConverter.ToUInt32(chunk, off + 0); if (!entriesByEntryId.TryGetValue(recordEntryId, out FdrEntry e)) { // Sidecar can carry entries not in the (possibly // compacted) caller dict — that's expected for // --task SecondPassFDR where compaction has already // dropped failing precursors. Skip silently. - continue; + return true; } - e.Score = BitConverter.ToDouble(data, off + 4); - e.RunPrecursorQvalue = BitConverter.ToDouble(data, off + 12); - e.RunPeptideQvalue = BitConverter.ToDouble(data, off + 20); + e.Score = BitConverter.ToDouble(chunk, off + 4); + e.RunPrecursorQvalue = BitConverter.ToDouble(chunk, off + 12); + e.RunPeptideQvalue = BitConverter.ToDouble(chunk, off + 20); // The EXPERIMENT-scope half, for the records THIS file's sidecar carries and no // others (format v5, issue #4486). Scoping it to the matched records is the // whole point: those columns are keyed by entry_id for the analysis, so applying @@ -788,6 +715,97 @@ public static bool TryReadOverlay(string path, e.ExperimentPeptideQvalue = exp.ExperimentPeptideQvalue; e.ExperimentAggregateScore = exp.ExperimentAggregateScore; } + return true; + }); + } + + /// + /// Validate a per-file sidecar's 32-byte header and walk its body in bounded chunks, + /// handing each record to as a (buffer, offset) pair. + /// Returns false on a missing or unreadable file, a header this build cannot consume, + /// a size that disagrees with the declared record count, a short read, or an + /// that returns false. + /// + /// The chunking is the point. Both callers used to take the body as one + /// File.ReadAllBytes array, and on a 446-run CHS analysis each run's 1st-pass + /// sidecar holds ~3.8 M records - 106 MB - which Stage 7's pool rebuild reads for every + /// run from inside a Parallel.For, several live at once, to land the ~648 K + /// records per run that survive compaction. Stage 7's memory band is Server-GC retained + /// COMMITTED memory rather than live data, so a parade of 106 MB large-object arrays + /// inflates it directly even though none of them is reachable for long. Reading the + /// same bytes in -record buffers gives the identical + /// result in the identical order with nothing on the large object heap. + /// + /// The per-record delegate is not a new cost of the same kind: every caller + /// already pays a dictionary probe per record, which dwarfs it, and + /// has always taken a per-record predicate. What was removed + /// is an allocation proportional to the FILE; what was added is proportional to + /// nothing. + /// + private static bool TryWalkRecords(string path, Pass expectedPass, + Func onRecord) + { + // NOT a bare catch: an OutOfMemoryException here is reported as a MISSING + // sidecar, and a missing 1st-pass sidecar leaves those entries at Score 0.0. + // The decoy side is not q-gated, so the zeros then compete in the picked- + // protein null and the run exits 0 with corrupted protein numbers. Let it + // propagate and kill the run instead (#4615 review). The whole-file array that + // made an OOM plausible here is gone, but the filter stays: it is about what a + // false return MEANS to the caller, not about how large the allocation was. + try + { + using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + long len = fs.Length; + if (len < HeaderLength) + return false; + var header = new byte[HeaderLength]; + if (!ReadFully(fs, header, HeaderLength)) + return false; + for (int i = 0; i < Magic.Length; i++) + { + if (header[i] != Magic[i]) + return false; + } + if (header[8] != FormatVersion) + return false; + // Reject mismatched pass bytes so a 2nd-pass sidecar can never + // be silently loaded into 1st-pass stubs (or vice versa) — the + // q-values would scramble without any visible error. + if (header[9] != (byte)expectedPass) + return false; + // bytes 10..16 reserved, ignored + ulong headerCount = BitConverter.ToUInt64(header, 16); + // Reject sidecars whose declared count exceeds physical + // record capacity. (headerCount can validly be < the caller's + // entry count — see the remarks on the callers for the + // pre-gap-fill / post-compaction cases.) Use checked + // arithmetic so a corrupt or malicious sidecar with a huge + // headerCount is rejected loudly instead of wrapping int + // silently. + if (!TryComputeExpectedLen(headerCount, out int expectedLen)) + return false; + if (len != expectedLen) + return false; + var chunk = new byte[RECORDS_PER_CHUNK * RecordLength]; + int remaining = (int)headerCount; + while (remaining > 0) + { + int take = Math.Min(RECORDS_PER_CHUNK, remaining); + if (!ReadFully(fs, chunk, take * RecordLength)) + return false; + remaining -= take; + for (int rec = 0; rec < take; rec++) + { + if (!onRecord(chunk, rec * RecordLength)) + return false; + } + } + } + } + catch (Exception ex) when (!(ex is OutOfMemoryException)) + { + return false; } return true; } diff --git a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs index 0fbb7b29fa..f8ef53abda 100644 --- a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs @@ -3748,6 +3748,76 @@ public void TestFdrScoresSidecarSupersetEntries() } } + /// + /// Chunk-boundary coverage for the buffered body reads. Both + /// + /// and now walk the body in fixed-size + /// buffers instead of materialising the file, which gives the record loop seams the + /// old whole-file indexing did not have: a count that is an exact multiple of the + /// buffer, one either side of it, and one that leaves a short final chunk. + /// + /// Every other sidecar test in this file writes a handful of records, so all of + /// them would pass against a reader that dropped, duplicated or misaligned every + /// record past the first buffer. That is the whole reason this one exists. + /// + /// The counts bracket a range of plausible buffer sizes rather than naming the + /// private constant, so the test keeps its meaning if the buffer is ever retuned. + /// + [TestMethod] + public void TestFdrScoresSidecarChunkBoundaries() + { + string dir = Path.Combine(Path.GetTempPath(), "fdr_sidecar_chunk_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + foreach (int count in new[] { 0, 1, 1023, 1024, 1025, 2047, 2048, 2049, 4096, 4103 }) + { + // A distinct path per count: FdrScoresSidecar refuses to write the same + // path twice in one process, which is the guard that keeps a sidecar + // write-once (P11). + string path = Path.Combine(dir, "n" + count + ".1st-pass.fdr_scores.bin"); + var written = new List(count); + for (int i = 0; i < count; i++) + written.Add(MakeFdrEntry((uint)i, -i * 0.5, i * 1.0e-6, 0.0)); + FdrScoresSidecar.Write(path, written, FdrScoresSidecar.Pass.FirstPass); + + var loaded = new List(count); + for (int i = 0; i < count; i++) + loaded.Add(MakeFdrEntry((uint)i, 0.0, 0.0, 0.0)); + Assert.IsTrue(FdrScoresSidecar.TryRead(path, loaded, FdrScoresSidecar.Pass.FirstPass), + "TryRead rejected a " + count + "-record sidecar"); + // Exact equality (delta 0.0): the assert recomputes the same expressions + // MakeFdrEntry used, so anything but a bit-identical round trip is a + // misaligned read rather than arithmetic drift. + for (int i = 0; i < count; i++) + { + Assert.AreEqual(-i * 0.5, loaded[i].Score, 0.0, + "Score at record " + i + " of " + count); + Assert.AreEqual(i * 1.0e-6, loaded[i].RunPrecursorQvalue, 0.0, + "RunPrecursorQvalue at record " + i + " of " + count); + Assert.AreEqual(i * 1.0e-6 + 1.0e-9, loaded[i].RunPeptideQvalue, 0.0, + "RunPeptideQvalue at record " + i + " of " + count); + } + + var byId = new Dictionary(); + for (int i = 0; i < count; i++) + byId[(uint)i] = MakeFdrEntry((uint)i, 0.0, 0.0, 0.0); + Assert.IsTrue( + FdrScoresSidecar.TryReadOverlay(path, byId, FdrScoresSidecar.Pass.FirstPass), + "TryReadOverlay rejected a " + count + "-record sidecar"); + for (int i = 0; i < count; i++) + { + Assert.AreEqual(-i * 0.5, byId[(uint)i].Score, 0.0, + "Overlay Score at record " + i + " of " + count); + } + } + } + finally + { + try { Directory.Delete(dir, true); } catch (IOException) { } + } + } + /// /// If a sidecar record's entry_id has no match in the caller's /// stub list, the reader must refuse rather than silently dropping From 9a1eb514c18735fb913c3f2566ccc31731e74c37 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Sun, 6 Sep 2026 13:43:28 -0700 Subject: [PATCH 02/30] Added the per-file survivor source Stage 7 needs to fold without holding the run * Extracted BuildRescoredPool's loop body as MaterializeRescoredFile, which brings ONE file's list to its post-rescore state. Nothing in it reads another file's entries, so per-file calls in buffer order are the same work in the same order as the whole-run build - the equivalence a byte-identical streamed Stage 7 rests on * Added RescoredEntries.StreamFiles: materializes each file on arrival, yields it, then clears and trims it. Peak is one file's survivors instead of every file's - measured 2026-09-06 at 446 CHS runs, the resident pool IS the Stage 7 peak (381 files x ~648 K survivors x 274 B = 68 GB, against a 68.0 GB managed peak). Re-enumerable, because a fold-then-apply consumer needs two passes * Made Value THROW after a stream. The lists are empty rather than unbuilt at that point, and PerFileEntries already names the failure a silent read would cause: one empty list per file, no exception, and a blib with no precursors * Handed the per-file source over only when a survivor loader exists. A run that kept its resident buffer cannot rebuild a file it dropped, so streaming there would destroy the only copy on its way past - StreamFiles walks the resident buffer instead, which is what the OSPREY_STAGE6_STREAM_SURVIVORS=0 A/B oracle needs * Added FileNames / FileCount that do not pull, and moved the stale-parquet scan onto them so a directory Stage 7 refuses outright no longer builds 289 M survivors it will discard No consumer streams yet, so this run's output is unchanged; the Stage 7 middle is converted next. TestStreamFilesDropsEachFileAndRefusesALaterValueRead pins the drop, the re-enumeration and the refusal. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 96 +++++++++++----- .../Osprey/Osprey.Tasks/PipelineByproducts.cs | 103 ++++++++++++++++-- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 35 +++--- .../Osprey.Test/ByproductContextTest.cs | 91 +++++++++++++++- 4 files changed, 268 insertions(+), 57 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index 268f43e3a8..cd506d241c 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -312,9 +312,32 @@ public override bool Run(PipelineContext ctx) // and parked in _poolPlan; deferring the decision as well as the work would // read state that is no longer true by the time the pull comes (see // RescoredPoolPlan). - var rescored = new RescoredEntries(_perFileEntries, () => BuildRescoredPool(ctx)); + // + // The per-file source is handed over only when a loader exists, and that condition + // is not a detail: a run that kept the resident buffer has no way to rebuild a file + // it dropped, so streaming there would destroy the only copy of the survivors on + // its way past. Null leaves StreamFiles walking the resident buffer, which is what + // the OSPREY_STAGE6_STREAM_SURVIVORS=0 A/B oracle needs it to do. + Action> materializeOneFile = null; + if (survivorLoader != null) + materializeOneFile = MaterializeOneFile; + var rescored = new RescoredEntries(_perFileEntries, () => BuildRescoredPool(ctx), + materializeOneFile); ctx.Publish(rescored); + void MaterializeOneFile(string fileName, List entries) + { + // Same refusal as BuildRescoredPool, and for the same reason: every guess + // available before Run has decided produces a wrong reported set rather than + // an error. + if (_poolPlan == null) + { + throw new InvalidOperationException( + @"RescoredEntries was streamed before PerFileRescoring decided how to build the survivor pool."); + } + MaterializeRescoredFile(ctx, _poolPlan, fileName, entries); + } + // Self-gate: rescore + reconciliation only run when there is // planning state to act on AND the rescore hasn't already been // done upstream. State comes from either FirstPassFdrTask's @@ -2475,33 +2498,7 @@ private void BuildRescoredPool(PipelineContext ctx) foreach (var kv in plan.Buffer) { progress.Report(++done); - // ONE parquet, not two. When this file's reconciled parquet was judged - // current, it already holds the survivor subset with Stage 6's boundaries - // applied and the gap-fill rows merged - so reading it makes both the - // Stage 4 read and the overlay that put those values back unnecessary - // (#4486). Stage 6 originally OVERWROTE the Stage 4 parquet, which is why - // one read used to give both; splitting the files left Stage 7 reading one - // for the rows and the other for the values. - string reconciledPath = null; - plan.ReconciledPaths?.TryGetValue(kv.Key, out reconciledPath); - bool loadedReconciled = reconciledPath != null && kv.Value.Count == 0; - MaterializeFileSurvivors(kv.Key, kv.Value, plan.Loader, ctx, - loadedReconciled ? reconciledPath : null); - if (plan.RescoredFiles == null) - continue; - // BEFORE the overlay, which appends gap-fill rows: the planner's indices - // address the survivor list as loaded, and appending shifts nothing but - // would be indexed if the reset ran after. The overlay preserves Score / - // q-values, so the reset survives it. - ResetRescoredTargetsForFile(plan, kv.Key, kv.Value); - // Skipped when the rows CAME from the reconciled parquet: the overlay - // would re-apply boundaries the rows already carry and append a second - // copy of the gap-fill rows already merged into them. - if (!loadedReconciled) - { - OverlayReconciledIntoFile(kv.Key, kv.Value, plan.ReconciledPaths, - plan.GapFill?.Value, canonicalize: false); - } + MaterializeRescoredFile(ctx, plan, kv.Key, kv.Value); } } sw.Stop(); @@ -2509,6 +2506,49 @@ private void BuildRescoredPool(PipelineContext ctx) sw.Elapsed.TotalSeconds, plan.Buffer.Count)); } + /// + /// Bring ONE file's survivor list to its post-rescore state: load, reset the rescored + /// targets, overlay the reconciled values. The body of 's + /// loop, extracted so a consumer can ask for one file at a time and DROP it - which is + /// the seam that loop's comment has described as needed since #4486. + /// + /// Nothing here reads another file's entries, so calling it per file in buffer + /// order is the same work in the same order as the whole-run build. That equivalence is + /// the reason a streamed Stage 7 can produce byte-identical output: the difference is + /// only how long each file's list stays alive. + /// + private void MaterializeRescoredFile(PipelineContext ctx, RescoredPoolPlan plan, + string fileName, List entries) + { + // ONE parquet, not two. When this file's reconciled parquet was judged + // current, it already holds the survivor subset with Stage 6's boundaries + // applied and the gap-fill rows merged - so reading it makes both the + // Stage 4 read and the overlay that put those values back unnecessary + // (#4486). Stage 6 originally OVERWROTE the Stage 4 parquet, which is why + // one read used to give both; splitting the files left Stage 7 reading one + // for the rows and the other for the values. + string reconciledPath = null; + plan.ReconciledPaths?.TryGetValue(fileName, out reconciledPath); + bool loadedReconciled = reconciledPath != null && entries.Count == 0; + MaterializeFileSurvivors(fileName, entries, plan.Loader, ctx, + loadedReconciled ? reconciledPath : null); + if (plan.RescoredFiles == null) + return; + // BEFORE the overlay, which appends gap-fill rows: the planner's indices + // address the survivor list as loaded, and appending shifts nothing but + // would be indexed if the reset ran after. The overlay preserves Score / + // q-values, so the reset survives it. + ResetRescoredTargetsForFile(plan, fileName, entries); + // Skipped when the rows CAME from the reconciled parquet: the overlay + // would re-apply boundaries the rows already carry and append a second + // copy of the gap-fill rows already merged into them. + if (!loadedReconciled) + { + OverlayReconciledIntoFile(fileName, entries, plan.ReconciledPaths, + plan.GapFill?.Value, canonicalize: false); + } + } + /// /// Every file whose .scores-reconciled.parquet is on disk and CURRENT for this /// run, as a file name -> path map. Asked while still holds the diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs index 7b3d4f2440..666b442fc7 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs @@ -402,6 +402,28 @@ internal abstract class PerFileEntries /// public virtual List>> Value => _buffer; + /// + /// The run's file names, in buffer order, WITHOUT pulling a deferred milestone. + /// + /// The keys are present from the moment the buffer is built and never change: + /// a deferred build fills each file's list IN PLACE - BuildRescoredPool walks + /// the very pairs it materializes into - and adds no pair. So a consumer that needs + /// only names can have them without materializing every file's survivors, which at + /// 446 CHS runs meant building 289 M entries to answer a question about 446 strings. + /// + /// Unlike this is safe to hand out. Names are + /// correct whether or not the entries are resident, so there is no state in which it + /// returns something that reads as valid and is not. + /// + public IReadOnlyList FileNames => _buffer.ConvertAll(kv => kv.Key); + + /// + /// The number of files in the buffer, WITHOUT pulling a deferred milestone, for the + /// callers that were reading only to take its Count. Same + /// reasoning as . + /// + public int FileCount => _buffer.Count; + /// /// The backing list as an OPAQUE reference, for identity comparison only - the DEBUG /// milestone-ordering guard in keys on which milestone @@ -502,19 +524,20 @@ internal sealed class RescoredEntries : PerFileEntries { private readonly Lazy _materialize; - /// The buffer already at its post-rescore state - nothing deferred. - public RescoredEntries(List>> value) : base(value) { } + /// Brings ONE file's list to its post-rescore state; null when this run has + /// no per-file source and the whole-run buffer is the only way to read entries. + private readonly Action> _materializeFile; /// - /// The run's file names, in buffer order. Reading them builds the buffer when the - /// build is still deferred, exactly as does - - /// every consumer of this milestone runs after Stage 7's own pool build, so by the - /// time anyone asks there is nothing left to defer. + /// Set once has dropped a file it materialized. From that + /// point the buffer's lists are EMPTY rather than unbuilt, and + /// refuses rather than handing them back - see the throw for why that distinction is + /// worth an exception. /// - public IReadOnlyList FileNames - { - get { return Value.ConvertAll(kv => kv.Key); } - } + private bool _streamed; + + /// The buffer already at its post-rescore state - nothing deferred. + public RescoredEntries(List>> value) : base(value) { } /// /// The run's files, one at a time, for a consumer that ITERATES and does not retain. @@ -541,17 +564,34 @@ public IEnumerable>> Files() /// state on the first read. Throws on failure - a deferred build /// has no return channel to the driver loop - and the throw is cached, so a second /// reader sees the same failure rather than a partially built pool. - public RescoredEntries(List>> value, Action materialize) + /// Brings ONE file's list to its post-rescore state, for + /// . Optional: without it streaming falls back to the + /// whole-run build, which is what the resident A/B oracle wants. + public RescoredEntries(List>> value, Action materialize, + Action> materializeFile = null) : base(value) { _materialize = new Lazy(() => { materialize(); return true; }, LazyThreadSafetyMode.ExecutionAndPublication); + _materializeFile = materializeFile; } public override List>> Value { get { + // A pull AFTER a stream would hand back the buffer's now-empty lists, which is + // the failure this type's base class calls out by name: no exception, no + // warning, and a blib with no precursors. The whole point of streaming is that + // those entries are gone on purpose, so there is no honest value to return and + // rebuilding silently would restore the very peak the stream exists to avoid. + if (_streamed) + { + throw new InvalidOperationException( + @"RescoredEntries.Value was read after StreamFiles dropped the survivor " + + @"pool. A consumer that runs after a streamed Stage 7 must fold through " + + @"StreamFiles too, or run before the stream starts."); + } // Reading Lazy.Value IS the build - once however many readers arrive, and a // failure cached and rethrown rather than retried. The bool it yields only // exists because Lazy needs a value type to hand back; discard it. @@ -559,6 +599,47 @@ public override List>> Value return base.Value; } } + + /// + /// The run's files one at a time, each materialized on arrival and DROPPED once the + /// consumer has folded it - the streamed source 's comment has been + /// waiting for. Peak is one file's survivors plus whatever the consumer accumulates, + /// instead of every file's at once: at 446 CHS runs that is ~0.2 GB against ~79 GB. + /// + /// Yields the buffer's own pairs, so a consumer that stamps entries stamps the + /// same objects it would have on the resident path. The stamps do not outlive the + /// yield, which is exactly why the per-file SIDECAR - not the entry - is what carries + /// Stage 7's results forward, as ComputePass2TransferCompeteFull documents. + /// + /// Re-enumerable: a second pass re-materializes each file from disk. Two passes + /// are the shape a fold-then-apply step needs (accumulate O(distinct) floors over every + /// file, then apply them), and paying a second read is the trade that removes the pool. + /// Falls back to the resident walk when this run has no per-file source, so the + /// oracle paths are unaffected. + /// + public IEnumerable>> StreamFiles() + { + if (_materializeFile == null) + { + foreach (var kv in Files()) + yield return kv; + yield break; + } + // base.Value, not Value: the pairs and their (empty) lists are what we materialize + // INTO, so reaching them must not trigger the whole-run build this method exists + // to replace - nor trip the _streamed guard on a second pass. + foreach (var kv in base.Value) + { + _materializeFile(kv.Key, kv.Value); + yield return kv; + // Dropped as soon as the consumer's foreach body returns. TrimExcess too: + // Clear leaves the backing array at its high-water capacity, which for a CHS + // file is ~648 K references still committed per file. + _streamed = true; + kv.Value.Clear(); + kv.Value.TrimExcess(); + } + } } /// diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 26979432e5..cead1f2910 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -295,13 +295,6 @@ public override bool Run(PipelineContext ctx) ProfilerHooks.LogManagedHeapAfterGcIfEnabled(ctx.LogInfo, @"stage7-inherited", string.Format(@"(post-GC, entering Stage 7, files={0})", nFiles)); - var perFileEntries = rescored.Value; - ProfilerHooks.LogManagedHeapAfterGcIfEnabled(ctx.LogInfo, @"stage7-pool", - string.Format(@"(post-GC, survivor pool built, files={0})", perFileEntries.Count)); - // Beside the probe that measures the pool, because it explains part of it: a - // distinct count still equal to the seed means the survivors' sequences are the - // library's own instances rather than one string per observation (#4486). - ctx.Get().LogSummary(ctx.LogInfo); var fullLibrary = ctx.Get().Value; var libraryById = ctx.Get().Value; var perFileParquetPaths = ctx.Get().Value; @@ -316,7 +309,13 @@ public override bool Run(PipelineContext ctx) // did not survive to be a feature the shipped Osprey needs, and keeping it meant // maintaining and testing a second read path onto a generation this branch // exists to retire. - var stale = StaleReconciledParquets(perFileEntries, perFileParquetPaths); + // + // Asked over the buffer's NAMES, and BEFORE the pool build below. The scan reads + // each file's parquet footer and needs no entries, so a directory this stage + // refuses outright no longer pays for 289 M survivors it is about to discard. + // Its own transients are footer metadata, which is why it can sit between the + // stage7-inherited and stage7-pool probes without distorting either. + var stale = StaleReconciledParquets(rescored.FileNames, perFileParquetPaths); if (stale.Count > 0) { throw new InvalidOperationException(string.Format( @@ -326,10 +325,18 @@ public override bool Run(PipelineContext ctx) "unusable, so a parquet-only rewrite would leave the directory " + "inconsistent. Re-run the analysis from Stage 5 over this directory. " + "Stale: [{2}].", - stale.Count, perFileEntries.Count, string.Join(", ", stale))); + stale.Count, rescored.FileCount, string.Join(", ", stale))); } - ReleaseUnscorableLibraryFragments(rescored, perFileEntries.Count, fullLibrary, ctx); + var perFileEntries = rescored.Value; + ProfilerHooks.LogManagedHeapAfterGcIfEnabled(ctx.LogInfo, @"stage7-pool", + string.Format(@"(post-GC, survivor pool built, files={0})", perFileEntries.Count)); + // Beside the probe that measures the pool, because it explains part of it: a + // distinct count still equal to the seed means the survivors' sequences are the + // library's own instances rather than one string per observation (#4486). + ctx.Get().LogSummary(ctx.LogInfo); + + ReleaseUnscorableLibraryFragments(rescored, rescored.FileCount, fullLibrary, ctx); // The 2nd-pass Percolator model, captured for the model-diagnostics // pass-2 model view; null when no reconciliation rescore happened. @@ -654,15 +661,15 @@ private static bool AnyReconciledParquet(OspreyConfig config) /// convert toward and has to be re-run from Stage 5 (issue #4486). /// private static List StaleReconciledParquets( - List>> perFileEntries, + IReadOnlyList fileNames, IReadOnlyDictionary perFileParquetPaths) { var stale = new List(); if (perFileParquetPaths == null) return stale; - foreach (var kv in perFileEntries) + foreach (var fileName in fileNames) { - if (!perFileParquetPaths.TryGetValue(kv.Key, out string scoresPath)) + if (!perFileParquetPaths.TryGetValue(fileName, out string scoresPath)) continue; string reconciledPath = ParquetScoreCache.ReconciledPathFromScoresPath(scoresPath); if (!File.Exists(reconciledPath)) @@ -680,7 +687,7 @@ private static List StaleReconciledParquets( StringComparison.Ordinal) || ParquetScoreCache.IsSubsetWithoutScoreIndex(reconciledPath)) { - stale.Add(kv.Key); + stale.Add(fileName); } } return stale; diff --git a/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs b/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs index 40d7972b6b..95f813841f 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs @@ -437,12 +437,95 @@ public void TestUndeferredMilestoneReadsStraightThrough() Assert.AreEqual(1, milestone.Value[0].Value.Count); } - private static List>> BufferWithOneFile() + /// + /// is the per-file source that lets Stage 7 + /// fold without holding the run: each file is materialized as the consumer reaches it + /// and DROPPED when the consumer moves on, so the peak is one file's survivors rather + /// than every file's at once. + /// + /// The last assertion is why this is a test and not a comment. After a stream the + /// buffer's lists are EMPTY rather than unbuilt, so a consumer that then read + /// Value would receive one empty list per file with no exception and no warning + /// - the blib-with-no-precursors failure warns about. It + /// has to throw instead, because there is no honest value to return and rebuilding + /// silently would restore the very peak the stream exists to avoid. + /// + [TestMethod] + public void TestStreamFilesDropsEachFileAndRefusesALaterValueRead() { - return new List>> + // No per-file source - the resident A/B oracle, where the run kept its buffer and + // has nothing to rebuild a dropped file from. StreamFiles falls back to the + // whole-run build and drops nothing. + var resident = BufferWithFiles(@"file1", @"file2"); + int residentBuilds = 0; + var residentMilestone = new RescoredEntries(resident, () => { - new KeyValuePair>(@"file1", new List()) - }; + residentBuilds++; + foreach (var kv in resident) + kv.Value.Add(new FdrEntry()); + }); + var residentWalk = new List(); + foreach (var kv in residentMilestone.StreamFiles()) + residentWalk.Add(kv.Key); + Assert.AreEqual(1, residentBuilds); + CollectionAssert.AreEqual(new[] { @"file1", @"file2" }, residentWalk); + Assert.AreEqual(2, ResidentEntryCount(resident), @"The fallback walk must not drop"); + Assert.AreSame(resident, residentMilestone.Value); + + // With a per-file source: the whole-run build never runs, and exactly ONE file is + // resident at any point in the walk - the property the whole change exists for. + var streamed = BufferWithFiles(@"file1", @"file2"); + int wholeRunBuilds = 0; + var materialized = new List(); + var milestone = new RescoredEntries(streamed, () => wholeRunBuilds++, + (fileName, entries) => + { + materialized.Add(fileName); + entries.Add(new FdrEntry()); + }); + var residentDuringWalk = new List(); + foreach (var kv in milestone.StreamFiles()) + { + Assert.AreEqual(1, kv.Value.Count, @"The current file must arrive materialized"); + residentDuringWalk.Add(ResidentEntryCount(streamed)); + } + Assert.AreEqual(0, wholeRunBuilds, @"A streamed walk must not build the whole-run pool"); + CollectionAssert.AreEqual(new[] { @"file1", @"file2" }, materialized); + CollectionAssert.AreEqual(new[] { 1, 1 }, residentDuringWalk); + Assert.AreEqual(0, ResidentEntryCount(streamed), @"The last file is dropped too"); + + // Re-enumerable, because a fold-then-apply consumer needs two passes: accumulate + // O(distinct) floors over every file, then apply them to every file. + foreach (var kv in milestone.StreamFiles()) + Assert.AreEqual(1, kv.Value.Count); + CollectionAssert.AreEqual(new[] { @"file1", @"file2", @"file1", @"file2" }, materialized); + + Assert.ThrowsException(() => milestone.Value); + } + + private static List>> BufferWithOneFile() + { + return BufferWithFiles(@"file1"); + } + + private static List>> BufferWithFiles( + params string[] fileNames) + { + var buffer = new List>>(); + foreach (string fileName in fileNames) + buffer.Add(new KeyValuePair>(fileName, new List())); + return buffer; + } + + /// Entries resident across the WHOLE buffer, which is what a streamed walk + /// must hold at one file's worth however many files the run has. + private static int ResidentEntryCount( + List>> buffer) + { + int count = 0; + foreach (var kv in buffer) + count += kv.Value.Count; + return count; } } } From 187c0a214c8db15e948a898096c43ce88974aeef Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Sun, 6 Sep 2026 17:39:36 -0700 Subject: [PATCH 03/30] Updated the workflow page and the boundary lists for what #4633 landed * Added the two experiment-wide artifacts #4633 introduced to every list that owes them. .1st-pass.retained_base_ids.bin is written by FirstPassFDR when Stage 6 planning ends and read by every run's compaction; .1st-pass.stratum.json is the protein-compact stratum, split out because a different phase produces it (P7). Both were absent from Osprey-workflow.html, which #4633 did not touch, and from doc 00's Boundary 2 -> 3 and 3 -> 4 lists * Added .2nd-pass.fdr_scores.bin to SecondPassFDR's INPUTS on the workflow page. It was listed only as an output "(where no worker ran)", which leaves out the ordinary case where the rescore worker produced it and the join folds it instead of recomputing * Cleared doc 00's in-flight item 2 and the Boundary 2 -> 3 caveat that told a reader to stage the whole cohort's envelopes because retained_base_ids.bin "does not exist on master". It does; the caveat was steering people around a fixed problem * Corrected two relay counts that named a fixed number of experiment-wide files. There are four now, and a count in prose is what went stale here in the first place Documentation only - no code, no behaviour change. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- pwiz_tools/Osprey/Osprey-workflow.html | 14 ++++---- .../Osprey/docs/00-pipeline-architecture.md | 35 +++++++++---------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey-workflow.html b/pwiz_tools/Osprey/Osprey-workflow.html index df0d33b478..55f59c22af 100644 --- a/pwiz_tools/Osprey/Osprey-workflow.html +++ b/pwiz_tools/Osprey/Osprey-workflow.html @@ -396,9 +396,9 @@

Osprey DIA pipeline workflow

▸ --task FirstPassFDR join · 1 node · holds O(distinct), never O(runs x entries) in  <stem>.scores.parquet, <stem>.calibration.json (every run, via --input-scores) - out <stem>.1st-pass.fdr_scores.bin, <stem>.reconciliation.json (Stage 5→6 boundary) · validity <out>.FirstPassFDR.osprey.task - out <blib-stem>.1st-pass.fdr_experiment.bin, <stem>.1st-pass.model.json (frozen pass-1 model, for the frozen pass-2 modes) - relay: both experiment-wide files to EVERY downstream node + out <stem>.1st-pass.fdr_scores.bin, .1st-pass.stratum.json, <stem>.reconciliation.json · validity <out>.FirstPassFDR.osprey.task + out <blib-stem>.1st-pass.fdr_experiment.bin, .1st-pass.retained_base_ids.bin (compaction key), <stem>.1st-pass.model.json + relay: every experiment-wide file to EVERY downstream node @@ -463,9 +463,9 @@

Osprey DIA pipeline workflow

▸ --task PerFileRescoring per-run fan-out · reads its own runs + the experiment baseline only in  <stem>.scores.parquet (via --input-scores) + .1st-pass.fdr_scores.bin, .reconciliation.json, .calibration.json, .spectra.bin - in  <blib-stem>.1st-pass.fdr_experiment.bin, <stem>.1st-pass.model.json · loaded once = the resident baseline + in  <blib-stem>.1st-pass.fdr_experiment.bin, .1st-pass.retained_base_ids.bin, <stem>.1st-pass.model.json · resident baseline out <stem>.scores-reconciled.parquet, .2nd-pass.fdr_decoys.bin, .2nd-pass.fdr_scores.bin (pass-2 worker) · validity <out>.PerFileRescoring.osprey.task - relay: the run's own set + 2 experiment-wide files + relay: the run's own set + every experiment-wide file @@ -506,8 +506,8 @@

Osprey DIA pipeline workflow

▸ --task SecondPassFDR join · 1 node · final aggregation - in  <stem>.scores-reconciled.parquet (via --input-scores; falls back to <stem>.scores.parquet for no-work runs) - in  <blib-stem>.1st-pass.fdr_experiment.bin, <stem>.1st-pass.model.json, <stem>.reconciliation.json, .calibration.json + in  <stem>.scores-reconciled.parquet (via --input-scores; falls back to <stem>.scores.parquet), .2nd-pass.fdr_scores.bin (worker) + in  <blib-stem>.1st-pass.fdr_experiment.bin, <stem>.1st-pass.model.json, .1st-pass.stratum.json, <stem>.reconciliation.json, .calibration.json out <output>.blib, <blib-stem>.2nd-pass.fdr_experiment.bin, <stem>.2nd-pass.fdr_scores.bin (where no worker ran) · validity <out>.SecondPassFDR.osprey.task relay: every run's reconciled set, with its .osprey.task stamps diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index cd807677f2..6da1410fb1 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -917,12 +917,6 @@ case they are the data and must be preserved. ### Boundary 2 -> 3: `FirstPassFDR` to `PerFileRescoring` -> **In flight** - this list is not yet one a single-run node can run on. The artifact that -> lets a fan-out worker obtain the join-wide compaction set without surveying the batch, -> `.1st-pass.retained_base_ids.bin`, does not exist on master; a node staged -> with exactly the list below derives a different survivor set rather than erroring. See -> item 2 under `## In flight`. Stage the whole cohort's envelopes until it lands. - Each rescore node needs its own runs' artifacts plus the experiment-wide set: Per run, for each run in the node's batch: @@ -934,13 +928,20 @@ Per run, for each run in the node's batch: Experiment-wide, to **every** node: - `.1st-pass.fdr_experiment.bin` +- `.1st-pass.retained_base_ids.bin` - the join-wide compaction set, written when + Stage 6 planning ends. It is what makes this list one a single-run node can actually run + on: without it a node would rebuild the union from every run's `reconciliation.json`, + which is the O(runs) pre-pass P6 forbids. Its absence is FATAL rather than silently + rebuilt, deliberately - see `ScoringTaskShared.ReadRetainedBaseIds` - `.1st-pass.model.json` (any one copy) - **mandatory on an ordinary run**, because the default pass-2 mode is a frozen one (`protein-compact`); an unset `OSPREY_PASS2_QVALUE` is not an opt-out +- `.1st-pass.stratum.json` (any one copy) - the protein-compact stratum, split out of + the model sidecar because a different phase produces it (P7) This is the boundary where a missing experiment-wide file does the most damage, because the node can often proceed without it and produce a plausible wrong answer rather than -failing. Ship both experiment-wide artifacts together or neither. +failing. Ship the experiment-wide artifacts together or none of them. ### Boundary 3 -> 4: `PerFileRescoring` to `SecondPassFDR` @@ -958,6 +959,9 @@ Per run, for **every** run in the cohort: Experiment-wide: - `.1st-pass.fdr_experiment.bin` - `.1st-pass.model.json` (any one copy) +- `.1st-pass.stratum.json` (any one copy) - the protein-compact stratum. The default + pass-2 mode reloads both this and the model here when neither was published in-process, + which is every distributed `SecondPassFDR` node - `.1st-pass.model-diagnostics.json`, when `--model-diagnostics` is on - the pass-1 half of the report exists nowhere else by this point @@ -1001,17 +1005,12 @@ the text says so rather than describing the current shape as though it were the protein FDR ends. See "The protein-q split, and the rule behind it" in `ai/todos/active/TODO-20260901_osprey_firstpassfdr_resume.md`. -2. **Two experiment-wide artifacts arrive with the fan-out fix**, and both are relay - obligations the checklist below will gain. `.1st-pass.retained_base_ids.bin` - is written by `FirstPassFDR` when Stage 6 planning ends and carries the join-wide - first-pass base ids unioned with every planned action target - sorted `uint32`, - library-bounded (1.49 MB at 373,487 ids). It is what lets a fan-out worker obtain the - compaction set without surveying the batch (P6), and it is the artifact that makes the - single-run input contract executable: today's Boundary 2 -> 3 list is not one a - one-run node can actually run on. `.1st-pass.stratum.json` is the P12 split - described under "Rows that are not what they look like". Both are on - `Skyline/work/20260901_osprey_firstpass_resume`; when it lands, "two experiment-wide - artifacts that must relay together" becomes four. +2. ~~**Two experiment-wide artifacts arrive with the fan-out fix.**~~ **LANDED** in + [#4633](https://github.com/ProteoWizard/pwiz/pull/4633) (`c4921f3d6c`, 2026-09-06). + `.1st-pass.retained_base_ids.bin` and `.1st-pass.stratum.json` are both + written by `FirstPassFDR` at the end of the phase that computes them, and both are now + carried in the Boundary 2 -> 3 and 3 -> 4 lists above. The count of experiment-wide + artifacts that must relay together is four. 3. **The bounded-loop shape of `PerFileRescoring`.** The task is still entered with a materialised all-runs entry list, and its baseline still carries maps keyed by run whose From ad4ef8d106ce7a1a3256e95b548d1501deaad2a7 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Sun, 6 Sep 2026 19:00:41 -0700 Subject: [PATCH 04/30] Removed the second-pass retrain toggle and transfer-compete, leaving two pass-2 modes * Removed OSPREY_PROTEIN_COMPACT_RETRAIN. It was the last way to reach a second-pass retrain, and the question it existed to measure is settled: retraining on the compaction-depleted pool is anti-conservative (1.57% true FDP at a nominal 1% on Stellar libdecoy entrapment against 0.92% for the pass-1 q, ~9% at 82 files). Issue #4484, closed * Removed OSPREY_PASS2_QVALUE=transfer-compete for a related but distinct reason, recorded in doc 12 rather than only here: its competition ran over a TARGET-CONDITIONED subset - survivors selected by per-run q on the target side, decoys admitted only by base_id pairing - which strips the decoys that WON the first-pass competition and so improves pass-2 q with no added evidence. 1.96% true FDP accepting 34,325 on 82-file SEA-AD, against 1.53% accepting 37,624 for protein-compact: dominated on both axes * Left protein-compact and transfer as the only modes. protein-compact carries the same paired-subsetting bias (#4581, open, with #4560 alongside) but also brings real protein-level evidence, which transfer-compete did not * Collapsed the full-population arm of ComputePass2TransferCompeteFull: its stratum is now required rather than optional, so the mode ternaries and the null-guard on the stratum lookup go with it * Recorded the consequence that simplifies what follows: THERE IS NO SECOND-PASS MODEL. The linear model the first pass trained is the pass-2 model unchanged - only the score distributions differ, because pass 2 runs on a subset Also cleared the #4633 staleness these files carried: doc 12 still said the protein stratum was "moving out of the model sidecar" and credited .1st-pass.model.json with carrying it. No behaviour change on either surviving mode; the removed paths were unreachable without setting the removed variables. 605 unit tests, zero-warning inspection. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Core/OspreyEnvironment.cs | 60 ++++------ .../Osprey/Osprey.Tasks/FirstPassFdrTask.cs | 6 +- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 112 +++++++----------- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 14 +-- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 5 +- pwiz_tools/Osprey/Osprey.Test/FdrTest.cs | 10 +- .../Osprey/Osprey.Test/TaskValidityKeyTest.cs | 1 - pwiz_tools/Osprey/Osprey/Program.cs | 13 +- pwiz_tools/Osprey/docs/07-fdr-control.md | 8 -- pwiz_tools/Osprey/docs/12-second-pass-fdr.md | 97 ++++++++------- pwiz_tools/Osprey/docs/20-command-line.md | 2 +- pwiz_tools/Osprey/regression.ps1 | 4 +- 12 files changed, 150 insertions(+), 182 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs b/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs index ca05158968..5d7d17cee5 100644 --- a/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs +++ b/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs @@ -402,16 +402,9 @@ private static int ResolveGbtMaxIterations() /// and map it to a q via the full pre-compaction 1st-pass score->q table. public const string PASS2_QVALUE_TRANSFER = @"transfer"; - /// The transfer-with-competition mode: score the - /// reconciled targets+decoys with the FROZEN 1st-pass model (no retrain), then - /// recompute q + PEP by a fresh target-decoy competition over that full reconciled - /// population (a non-depleted null) -- i.e. the frozen weights feed the standard - /// competition q/PEP math instead of a co-monotone score->q table lookup. - public const string PASS2_QVALUE_TRANSFER_COMPETE = @"transfer-compete"; - - /// The protein-anchored constrained mode: like - /// (frozen 1st-pass model, no retrain), - /// but the target-decoy competition is CONSTRAINED to the peptides of proteins + /// The protein-anchored constrained mode: the FROZEN + /// 1st-pass model (no retrain), with the target-decoy competition CONSTRAINED to the + /// peptides of proteins /// detected in the 1st pass -- included as target+decoy PAIRS so the stratum's null /// stays fair. Removing off-stratum decoys from the null lowers q for stratum /// members (reduced multiple testing / independent filtering; Bourgon 2010), which @@ -473,14 +466,26 @@ private static int ResolveGbtMaxIterations() /// Unset normalizes to the default; an unrecognized value is a startup ERROR (see /// ). Read once at process start. /// - /// The former default percolator - retrain the 2nd-pass Percolator SVM and - /// recompute a target/decoy null on the reconciled + COMPACTED pool - was REMOVED, not - /// merely demoted. Compaction strips most decoys from that pool, so the retrained null - /// is thin and the reported q anti-conservative: 1.57% true FDP at a nominal 1% on - /// Stellar libdecoy entrapment (vs 0.92% for the 1st-pass q), and ~9% on an 82-file - /// SEA-AD set. The linear model trained by the 1st-pass SVM is now the model for pass 2 - /// in every mode; only the diagnostic A/B still - /// retrains. See ai/todos/active/TODO-20260710_osprey_pass2_recalibration_fix.md. + /// SECOND-PASS RETRAINING IS GONE, and these two modes are what remain. The former + /// default percolator - retrain the 2nd-pass Percolator SVM and recompute a + /// target/decoy null on the reconciled + COMPACTED pool - was REMOVED, not merely + /// demoted. Compaction strips most decoys from that pool, so the retrained null is thin + /// and the reported q anti-conservative: 1.57% true FDP at a nominal 1% on Stellar + /// libdecoy entrapment (vs 0.92% for the 1st-pass q), and ~9% on an 82-file SEA-AD set. + /// The OSPREY_PROTEIN_COMPACT_RETRAIN A/B toggle that reached it followed. So did + /// transfer-compete, for a related but distinct reason: its competition ran over a + /// TARGET-CONDITIONED subset - survivors chosen by target per-run q, decoys admitted only + /// by base_id pairing - which strips decoys that WON the 1st-pass competition and so + /// improves pass-2 q with no added evidence. Measured 1.96% true FDP at a nominal 1% on + /// 82-file SEA-AD against 1.53% for the default, with fewer ids: dominated on both axes. + /// See issues #4484 (closed) and #4581 (open, the same bias in the surviving default), and + /// docs/12-second-pass-fdr.md, "Why a second-pass null is a problem". + /// + /// The consequence worth naming, because it simplifies everything downstream: THERE IS + /// NO SECOND-PASS MODEL. The linear model the 1st-pass SVM trained is the model for + /// pass 2, unchanged - only the score DISTRIBUTIONS differ, because pass 2 runs on a + /// subset. Anything that used to ask a retrained pass-2 model for its weights can read + /// the frozen ones instead. /// /// Switching modes within one output directory is now SAFE: the mode participates in /// the resume validity key through , so a @@ -506,25 +511,11 @@ private static int ResolveGbtMaxIterations() public static readonly bool Pass2TransferQ = string.Equals(Pass2QValue, PASS2_QVALUE_TRANSFER, StringComparison.Ordinal); - /// True when selects the frozen-model + - /// target-decoy competition path (OSPREY_PASS2_QVALUE=transfer-compete). - public static readonly bool Pass2TransferCompete = - string.Equals(Pass2QValue, PASS2_QVALUE_TRANSFER_COMPETE, StringComparison.Ordinal); - /// True when selects the protein-anchored /// constrained competition (OSPREY_PASS2_QVALUE=protein-compact). public static readonly bool Pass2ProteinCompact = string.Equals(Pass2QValue, PASS2_QVALUE_PROTEIN_COMPACT, StringComparison.Ordinal); - /// Diagnostic A/B toggle (OSPREY_PROTEIN_COMPACT_RETRAIN): when set with - /// OSPREY_PASS2_QVALUE=protein-compact, SKIP the frozen 1st-pass model + stratum - /// competition and instead RETRAIN the 2nd-pass Percolator over the same - /// stratum-expanded compacted pool. Isolates the frozen-vs-retrain FDR-calibration - /// difference (same reported set, only the 2nd-pass scoring changes) for the - /// FDRBench/entrapment oracle. Off (frozen) by default. - public static readonly bool Pass2ProteinCompactRetrain = - IsSetAndNotZero(@"OSPREY_PROTEIN_COMPACT_RETRAIN"); - /// /// OSPREY_PASS2_VERIFY_WORKER: re-run the per-file second-pass competition inside Stage 7 /// and assert it against the answer the rescore worker wrote (issue #4486). A TEST @@ -983,8 +974,6 @@ private static string NormalizePass2QValue(string raw) string v = raw.Trim().ToLowerInvariant(); if (v == PASS2_QVALUE_TRANSFER) return PASS2_QVALUE_TRANSFER; - if (v == PASS2_QVALUE_TRANSFER_COMPETE) - return PASS2_QVALUE_TRANSFER_COMPETE; if (v == PASS2_QVALUE_PROTEIN_COMPACT) return PASS2_QVALUE_PROTEIN_COMPACT; // An unrecognized token normalizes to the default only so the other statics are @@ -998,8 +987,7 @@ private static bool IsUnrecognizedPass2QValue(string raw) if (string.IsNullOrWhiteSpace(raw)) return false; string v = raw.Trim().ToLowerInvariant(); - return v != PASS2_QVALUE_TRANSFER && - v != PASS2_QVALUE_TRANSFER_COMPETE && v != PASS2_QVALUE_PROTEIN_COMPACT; + return v != PASS2_QVALUE_TRANSFER && v != PASS2_QVALUE_PROTEIN_COMPACT; } private static int ParseIntOrZero(string name) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs index adfae91a8d..324e368173 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs @@ -2527,8 +2527,7 @@ internal static FeatureContributions RunPercolatorFdr( // model instead of retraining. Null (a pure no-op in the engine) on the default // percolator path and on the 2nd-pass run, so scoring stays byte-identical. Action captureModel = null; - if ((OspreyEnvironment.Pass2TransferQ || OspreyEnvironment.Pass2TransferCompete || - OspreyEnvironment.Pass2ProteinCompact) && + if ((OspreyEnvironment.Pass2TransferQ || OspreyEnvironment.Pass2ProteinCompact) && string.Equals(passLabel, @"First-pass", StringComparison.Ordinal)) { // Publish is add-only (throws on a duplicate key); guard so a first pass @@ -2954,8 +2953,7 @@ int FlushPartialSidecar(string fileName, IReadOnlyList records) var reloadedModel = LoadCurrentModelSidecar(perFileParquetPaths, sidecarValidityKey); Action captureModel = results => { - if ((OspreyEnvironment.Pass2TransferQ || OspreyEnvironment.Pass2TransferCompete || - OspreyEnvironment.Pass2ProteinCompact) && + if ((OspreyEnvironment.Pass2TransferQ || OspreyEnvironment.Pass2ProteinCompact) && !ctx.TryGet(out _)) { // Stamp the arm THIS pass ran under; the 2nd pass may be another process. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index 6c0a18a356..6564bac2ef 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -112,9 +112,8 @@ internal static FeatureContributions ComputeAndPersist( // when the model is already present, the mode is the default retrain, or the // sidecar is absent (the existing fail-fast then applies). // protein-compact needs the ProteinCompactStratum too; it rides in the same - // sidecar, so one reload serves all three frozen modes. + // sidecar, so one reload serves both surviving modes. bool wantsFrozenModel = OspreyEnvironment.Pass2TransferQ || - OspreyEnvironment.Pass2TransferCompete || OspreyEnvironment.Pass2ProteinCompact; if (wantsFrozenModel && !ctx.TryGet(out _)) { @@ -158,9 +157,7 @@ internal static FeatureContributions ComputeAndPersist( // it was told to retrain. These own their whole per-file cycle (materialize, score, // compete, write the sidecar, drop the file), so they skip both the whole-pool // pass-1 scalar seed above and the resident sidecar write below. - bool frozenCompetition = - OspreyEnvironment.Pass2TransferCompete || - (OspreyEnvironment.Pass2ProteinCompact && !OspreyEnvironment.Pass2ProteinCompactRetrain); + bool frozenCompetition = OspreyEnvironment.Pass2ProteinCompact; // True once a path has written every file's .2nd-pass.fdr_scores.bin itself, which // is what the resident write block below tests before repeating the work. @@ -310,14 +307,10 @@ internal static FeatureContributions ComputeAndPersist( var swPass2 = Stopwatch.StartNew(); - // The frozen COMPETITION modes (transfer-compete, protein-compact) re-score - // with the frozen 1st-pass model over the full pre-compaction population / - // protein stratum - a competition the projection engine does not do (it - // trains + competes over the survivor set only). They own the whole per-file - // cycle: materialize, score, compete, write the sidecar, drop, so they need - // neither the projection engine nor a resident pool. - // protein-compact + OSPREY_PROTEIN_COMPACT_RETRAIN=1 is the exception: it - // retrains, so it stays on the projection (streaming-retrain) path. + // protein-compact re-scores with the frozen 1st-pass model over the + // protein stratum. It owns the whole per-file cycle: materialize, score, + // compete, write the sidecar, drop - so it needs neither the projection + // engine nor a resident pool. // // --model-diagnostics needs the resident 2nd-pass model: its feature // contributions feed the pass-2 model view, and the projection 2nd pass @@ -1464,10 +1457,10 @@ public Pass2ExperimentScope(FdrExperimentAccumulator accumulator) } /// - /// Run the frozen-model COMPETITION second pass (transfer-compete / protein-compact): - /// resolve the frozen 1st-pass model and, for protein-compact, its stratum, then hand - /// them to . Returns true when it ran and - /// wrote every file's 2nd-pass sidecar. + /// Run the frozen-model COMPETITION second pass (protein-compact): resolve the frozen + /// 1st-pass model and its stratum, then hand them to + /// . Returns true when it ran and wrote + /// every file's 2nd-pass sidecar. /// /// Fail-fast, because an explicitly requested frozen mode must NEVER silently /// degrade to the anti-conservative retrain. Absent inputs - the frozen 1st-pass model @@ -1475,8 +1468,9 @@ public Pass2ExperimentScope(FdrExperimentAccumulator accumulator) /// and skipped 1st-pass training, or a distributed SecondPassFDR node that never trained /// pass 1), or a missing / corrupt 1st-pass sidecar - mean the mode cannot be honored, /// so this aborts with actionable guidance rather than reporting looser FDR than a cold - /// straight-through run under the same mode. (protein-compact + - /// OSPREY_PROTEIN_COMPACT_RETRAIN=1 retrains by design and never reaches here.) + /// straight-through run under the same mode. There is no longer a retrain to degrade + /// TO - second-pass retraining was removed with issue #4484 - so an absent input is a + /// hard stop rather than a quieter answer. /// private static bool ComputePass2FrozenCompetition( PipelineContext ctx, @@ -1506,17 +1500,13 @@ private static bool ComputePass2FrozenCompetition( "model, 1st-pass scalar sidecars or protein stratum are absent, or a file's input " + "path could not be resolved - e.g. a warm " + "rerun or a distributed SecondPassFDR node that did not train pass 1 in-process). " + - "The warning above names which. Run the " + - "frozen modes on the straight-through path, rerun without the score cache, or unset " + - "OSPREY_PASS2_QVALUE for the default retrain{1}.", - OspreyEnvironment.Pass2QValue, - OspreyEnvironment.Pass2ProteinCompact - ? ", or set OSPREY_PROTEIN_COMPACT_RETRAIN=1 to retrain over the stratum" - : string.Empty)); + "The warning above names which. Run this mode on the straight-through path, or " + + "rerun without the score cache so the 1st pass trains in-process.", + OspreyEnvironment.Pass2QValue)); } /// - /// OSPREY_PASS2_QVALUE=transfer-compete (full-population form). Recompute the reported + /// OSPREY_PASS2_QVALUE=protein-compact. Recompute the reported /// precursor q-values + PEP by re-running the target-decoy competition over the ENTIRE /// 1st-pass population -- read as SCALARS from each file's persisted /// .1st-pass.fdr_scores.bin -- with ONLY the reconciled survivors' scores swapped @@ -1554,22 +1544,27 @@ private static bool ComputePass2TransferCompeteFull( PercolatorResults frozenModel, string pass1ExperimentAgg, Pass2SidecarWriter writer, - HashSet stratumBaseIds = null) + HashSet stratumBaseIds) { - // stratumBaseIds == null -> transfer-compete (full-population competition). - // non-null -> protein-compact: the competition is CONSTRAINED to the stratum - // (peptides of >=2-peptide 1st-pass proteins), and the map-back below leaves - // OFF-stratum survivors on their 1st-pass q (report = pass1 U stratum passers, - // so re-scoping only adds, never drops an already-passing peptide). - bool proteinCompact = stratumBaseIds != null; - string mode = proteinCompact ? "protein-compact" : "transfer-compete"; + // The competition is CONSTRAINED to the stratum (peptides of >=2-peptide 1st-pass + // proteins), and the map-back below leaves OFF-stratum survivors on their 1st-pass q + // (report = pass1 U stratum passers, so re-scoping only adds, never drops an + // already-passing peptide). The full-population form this method also served was + // transfer-compete, removed because its competition ran over a target-conditioned + // subset - see the OspreyEnvironment.Pass2QValue remarks and issue #4581. + if (stratumBaseIds == null) + { + throw new ArgumentNullException(nameof(stratumBaseIds), + @"protein-compact is the only competition mode; its stratum is required."); + } + string mode = OspreyEnvironment.PASS2_QVALUE_PROTEIN_COMPACT; // Works for whichever classifier the 1st pass trained (linear SVM or - // gradient-boosted trees) -- the scorer hides that choice, so transfer-compete - // stays the honest-FDR path under --fdr-method gbdt too. + // gradient-boosted trees) -- the scorer hides that choice, so this stays the + // honest-FDR path under --fdr-method gbdt too. var scorer = FrozenModelScorer.TryCreate(frozenModel); if (scorer == null) { - ctx.LogWarning("transfer-compete: frozen 1st-pass model has no usable model/standardizer."); + ctx.LogWarning(mode + ": frozen 1st-pass model has no usable model/standardizer."); return false; } var sw = Stopwatch.StartNew(); @@ -1668,7 +1663,7 @@ private static bool ComputePass2TransferCompeteFull( { if (!perFileParquetPaths.TryGetValue(fileName, out string parquetPath)) { - ctx.LogWarning("transfer-compete: no parquet path for '" + fileName + + ctx.LogWarning(mode + ": no parquet path for '" + fileName + "'; cannot locate its 1st-pass scalar sidecar."); return false; } @@ -1699,7 +1694,7 @@ private static bool ComputePass2TransferCompeteFull( fileName + ".1st-pass.fdr_scores.bin"); if (!File.Exists(sidecarPath)) { - ctx.LogWarning("transfer-compete: 1st-pass scalar sidecar not found: " + sidecarPath); + ctx.LogWarning(mode + ": 1st-pass scalar sidecar not found: " + sidecarPath); return false; } // Existence was never enough. ReadScalars THROWS on bad magic, a stale version, a @@ -1712,7 +1707,7 @@ private static bool ComputePass2TransferCompeteFull( if (!FdrScoresSidecar.IsCurrentFormat(sidecarPath, FdrScoresSidecar.Pass.FirstPass)) { ctx.LogWarning( - "transfer-compete: 1st-pass scalar sidecar is not a readable v" + + mode + ": 1st-pass scalar sidecar is not a readable v" + FdrScoresSidecar.FormatVersion + " first-pass file: " + sidecarPath); return false; } @@ -1739,8 +1734,7 @@ private static bool ComputePass2TransferCompeteFull( "scores swapped in for up to {2} reconciled survivor observations - no retrain, one " + "file resident at a time{3}.", mode, fileKeys.Count, survivorObservations, - proteinCompact ? ", competition CONSTRAINED to the " + stratumBaseIds.Count + "-base_id protein stratum" - : ", full-population null")); + ", competition CONSTRAINED to the " + stratumBaseIds.Count + "-base_id protein stratum")); // This competition reduces per base_id by MAX, and BOTH modes that reach it then // overwrite the reported experiment q from that reduction. Neither is compatible with @@ -1782,19 +1776,14 @@ private static bool ComputePass2TransferCompeteFull( "from a MAX-aggregated competition, which {3}. Use OSPREY_PASS2_QVALUE={4}, " + "which carries the 1st-pass mean(best-N) q through unchanged, for a " + "mean(best-N) arm.", - proteinCompact - ? OspreyEnvironment.PASS2_QVALUE_PROTEIN_COMPACT - : OspreyEnvironment.PASS2_QVALUE_TRANSFER_COMPETE, + OspreyEnvironment.PASS2_QVALUE_PROTEIN_COMPACT, pass1Arm, armRecorded ? " (recorded in the 1st-pass model sidecar)" : " (INFERRED from this process's environment - the 1st-pass model sidecar " + "predates arm recording and does not say which arm trained it)", - proteinCompact - ? "would leave on-stratum precursors max-aggregated and off-stratum " + - "precursors on their 1st-pass mean(best-N) q - one column, two statistics" - : "would replace every precursor's mean(best-N) q with a max q, making the " + - "run indistinguishable from a default run in its own output", + "would leave on-stratum precursors max-aggregated and off-stratum " + + "precursors on their 1st-pass mean(best-N) q - one column, two statistics", OspreyEnvironment.PASS2_QVALUE_TRANSFER)); } @@ -2132,9 +2121,7 @@ void ApplyFileRunQ(string fileKey, StreamingFdr.FileCompetition contribution) // on-stratum recompute and off-stratum carry-through - read side by side. FdrExperimentRecord FinishRecord(FdrScoreRecord rec) { - // stratumBaseIds != null IS proteinCompact - written as the null test the branch - // actually depends on, so the guard is local to the dereference it protects. - if (stratumBaseIds != null && !stratumBaseIds.Contains(rec.EntryId & 0x7FFFFFFFu)) + if (!stratumBaseIds.Contains(rec.EntryId & 0x7FFFFFFFu)) { // Off-stratum survivors keep their 1st-pass EXPERIMENT q (report = pass1 U // stratum passers). That q is a pass-1 property anchored on the @@ -2293,21 +2280,12 @@ private static FeatureContributions ComputePass2Resident( // sequence (transfer-compete's frozen-model recompute, or a retrain) // regardless of which classifier the 1st pass trained. The frozen model // carried in ctx is whichever one that was, and the score passes select - // on it, so transfer-compete works unchanged for trees. + // on it, so the frozen competition works unchanged for trees. case FdrMethod.Percolator: case FdrMethod.Gbdt: - // OSPREY_PASS2_QVALUE=transfer-compete / protein-compact (frozen) are handled - // at the TOP of ComputePass2Resident (before the resident feature reload) so - // their frozen score pass streams one file at a time -- see - // ComputePass2TransferCompeteFull. Only the retrain A/B toggle and - // OSPREY_PASS2_QVALUE=transfer reach here. - if (OspreyEnvironment.Pass2ProteinCompact && OspreyEnvironment.Pass2ProteinCompactRetrain) - { - ctx.LogInfo( - "OSPREY_PROTEIN_COMPACT_RETRAIN=1: skipping the frozen-model + stratum " + - "competition; RETRAINING the 2nd-pass over the stratum-expanded compacted pool " + - "(frozen-vs-retrain FDR A/B)."); - } + // protein-compact is handled by the frozen competition before the resident + // feature reload, so its score pass streams one file at a time. Only + // OSPREY_PASS2_QVALUE=transfer reaches here. // OSPREY_PASS2_QVALUE=transfer: instead of retraining a 2nd-pass SVM on // the decoy-depleted reconciled+compacted set (which re-derives an // anti-conservative experiment-scope q), carry the pass-1 q through and diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index cd506d241c..2a1ce42933 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -234,7 +234,7 @@ public override IEnumerable Outputs(PipelineContext ctx) // output this task will not write would make the driver's IsTaskAlreadyDone - which // requires EVERY declared output to exist - permanently false, re-running Stage 6 // on every resume. - if (!OspreyEnvironment.Pass2ProteinCompact && !OspreyEnvironment.Pass2TransferCompete) + if (!OspreyEnvironment.Pass2ProteinCompact) yield break; foreach (var input in ctx.Config.InputFiles) { @@ -996,7 +996,7 @@ private static Pass2PerFileWorker TryCreatePass2Worker( IReadOnlyDictionary perFileParquetPaths, OspreyConfig config, PipelineContext ctx, string taskName, string taskValidityKey) { - if (!OspreyEnvironment.Pass2ProteinCompact && !OspreyEnvironment.Pass2TransferCompete) + if (!OspreyEnvironment.Pass2ProteinCompact) return null; var sidecar = FirstPassModelIO.LoadFromAny(perFileParquetPaths); if (sidecar?.Model == null) @@ -1014,10 +1014,8 @@ private static Pass2PerFileWorker TryCreatePass2Worker( "model/standardizer, so the per-file half stays in SecondPassFDR for this run."); return null; } - // protein-compact competes within the stratum; transfer-compete over the full - // population. Mirrors ComputePass2TransferCompeteFull's own selector so the two - // cannot drift on which mode means which competition. - bool proteinCompact = OspreyEnvironment.Pass2ProteinCompact; + // protein-compact is the only competition mode - the guard above returned already + // if it was not selected - so the stratum is always the constraint. var inputByName = new Dictionary(StringComparer.Ordinal); if (config.InputFiles != null) { @@ -1062,8 +1060,8 @@ void WriteAnswer(string fileName, IReadOnlyList records, } return new Pass2PerFileWorker( scorer, - proteinCompact ? @"protein-compact" : @"transfer-compete", - proteinCompact ? sidecar.StratumBaseIds : null, + OspreyEnvironment.PASS2_QVALUE_PROTEIN_COMPACT, + sidecar.StratumBaseIds, Pass2FdrSidecar.LoadPass1ExperimentRecords(config), WriteAnswer, ctx.LogWarning); diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index cead1f2910..74ee82d641 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -115,7 +115,7 @@ public override IEnumerable Inputs(PipelineContext ctx) if (!string.IsNullOrEmpty(pass1Experiment)) yield return pass1Experiment; - if (!OspreyEnvironment.Pass2ProteinCompact && !OspreyEnvironment.Pass2TransferCompete) + if (!OspreyEnvironment.Pass2ProteinCompact) yield break; foreach (var input in ctx.Config.InputFiles) { @@ -181,8 +181,7 @@ public override IEnumerable Outputs(PipelineContext ctx) // via AnalysisPipeline.WriteTaskSidecars) and, worse, lets the driver's // IsTaskAlreadyDone - which requires every declared output to exist - skip THIS task // the moment Stage 6 has written them, which is the join never running at all. - bool workerOwnsPerFileSidecars = - OspreyEnvironment.Pass2ProteinCompact || OspreyEnvironment.Pass2TransferCompete; + bool workerOwnsPerFileSidecars = OspreyEnvironment.Pass2ProteinCompact; if (ctx.Config.InputFiles != null && !workerOwnsPerFileSidecars) { foreach (var input in ctx.Config.InputFiles) diff --git a/pwiz_tools/Osprey/Osprey.Test/FdrTest.cs b/pwiz_tools/Osprey/Osprey.Test/FdrTest.cs index 9ca7851f52..d3a27581d5 100644 --- a/pwiz_tools/Osprey/Osprey.Test/FdrTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/FdrTest.cs @@ -472,9 +472,9 @@ public void TestGbdtLearnsNonMonotoneFeature() /// /// must accept BOTH classifiers. This guards the - /// 2nd-pass transfer paths (OSPREY_PASS2_QVALUE=transfer / transfer-compete), - /// which decline when handed a model they cannot read and fall back to the - /// anti-conservative retrain. Before the scorer existed they read FoldWeights + /// 2nd-pass frozen paths (OSPREY_PASS2_QVALUE=transfer / protein-compact), + /// which decline when handed a model they cannot read - and since the retrain was + /// removed, declining is now a hard stop. Before the scorer existed they read FoldWeights /// directly, so a gbdt run would have silently taken that fallback -- honest /// FDR lost, with nothing failing. /// @@ -495,8 +495,8 @@ public void TestFrozenModelScorerAcceptsBothClassifiers() var treeModel = PercolatorTrainer.RunPercolator(entries, treeConfig); var treeScorer = FrozenModelScorer.TryCreate(treeModel); Assert.IsNotNull(treeScorer, - "frozen tree model must be scorable -- a null here silently drops " + - "transfer-compete back to the 2nd-pass retrain"); + "frozen tree model must be scorable -- a null here fails the frozen " + + "competition, which has no retrain left to fall back to"); Assert.IsTrue(treeScorer.IsGradientBoostedTrees); Assert.AreEqual(2, treeScorer.NumFeatures); diff --git a/pwiz_tools/Osprey/Osprey.Test/TaskValidityKeyTest.cs b/pwiz_tools/Osprey/Osprey.Test/TaskValidityKeyTest.cs index 1aee3eba3b..db782f65a6 100644 --- a/pwiz_tools/Osprey/Osprey.Test/TaskValidityKeyTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/TaskValidityKeyTest.cs @@ -177,7 +177,6 @@ private static void AssertEachArmKeysDifferently() var modes = new[] { OspreyEnvironment.PASS2_QVALUE_TRANSFER, - OspreyEnvironment.PASS2_QVALUE_TRANSFER_COMPETE, OspreyEnvironment.PASS2_QVALUE_PROTEIN_COMPACT }; for (int i = 0; i < modes.Length; i++) diff --git a/pwiz_tools/Osprey/Osprey/Program.cs b/pwiz_tools/Osprey/Osprey/Program.cs index e8e969b373..6c6f9c1076 100644 --- a/pwiz_tools/Osprey/Osprey/Program.cs +++ b/pwiz_tools/Osprey/Osprey/Program.cs @@ -303,12 +303,15 @@ static int Main(string[] args) if (OspreyEnvironment.Pass2QValueUnrecognized) { LogError(string.Format( - "OSPREY_PASS2_QVALUE is not a recognized mode. Recognized: '{0}', '{1}', " + - "'{2}'. Unset it for the default ('{2}'). The 'percolator' mode was " + - "REMOVED: it retrained the 2nd-pass SVM on a compaction-depleted decoy " + - "pool, which reports anti-conservative q-values.", + "OSPREY_PASS2_QVALUE is not a recognized mode. Recognized: '{0}', '{1}'. " + + "Unset it for the default ('{1}'). 'percolator' was REMOVED: it retrained " + + "the 2nd-pass SVM on a compaction-depleted decoy pool, which reports " + + "anti-conservative q-values. 'transfer-compete' was REMOVED for a related " + + "reason: it selected survivors by TARGET per-run q and admitted decoys only " + + "by pairing, stripping decoys that won the 1st-pass competition, so its q " + + "improved with no added evidence - 1.96% true FDP at a nominal 1% on 82-file " + + "SEA-AD, against 1.53% for the default, and with FEWER ids.", OspreyEnvironment.PASS2_QVALUE_TRANSFER, - OspreyEnvironment.PASS2_QVALUE_TRANSFER_COMPETE, OspreyEnvironment.PASS2_QVALUE_PROTEIN_COMPACT)); return 1; } diff --git a/pwiz_tools/Osprey/docs/07-fdr-control.md b/pwiz_tools/Osprey/docs/07-fdr-control.md index 20d8175ae1..0cbb2305ff 100644 --- a/pwiz_tools/Osprey/docs/07-fdr-control.md +++ b/pwiz_tools/Osprey/docs/07-fdr-control.md @@ -522,7 +522,6 @@ That makes the second-pass q-value mode | `OSPREY_PASS2_QVALUE` | Behavior after a mean(best-N) first pass | |---|---| | `transfer` | **The compatible mode.** Carries the first-pass q through unchanged, so the reported experiment q stays mean(best-N). | -| `transfer-compete` | **Refused** (`Pass2FdrSidecar` throws). It rewrites every survivor's experiment q from a MAX-aggregated competition, making a reproducibility-weighted run indistinguishable from a default run in its own output. | | `protein-compact` (default) | **Refused** (but see the caveat below). Worse than uniform: on-stratum survivors would get the MAX-aggregated value while off-stratum survivors keep their first-pass mean(best-N) q, giving one reported column with two statistics and no way for a consumer to tell which row used which. | **Because `protein-compact` is the DEFAULT, a mean(best-N) arm must set @@ -531,13 +530,6 @@ deliberate: the alternative - silently using `transfer` whenever the first pass would make the effective default depend on another variable, which is harder to reason about than a loud failure whose message names the fix. -> **Caveat: `protein-compact` + `OSPREY_PROTEIN_COMPACT_RETRAIN=1` is NOT refused.** The refusal -> lives in the frozen-model recompute, and that A/B lever deliberately bypasses it to retrain -> instead - so the combination retrains and silently reports a MAX-aggregated experiment q. This is -> left as-is rather than guarded because the combination is a three-way diagnostic opt-in, and -> these environment variables are development instrumentation rather than a supported interface -> (see `ai/docs/osprey-development-guide.md`). Do not read the "Refused" row above as covering it. - The refusal gates on the arm the **first pass recorded** - persisted as `ExperimentAgg` in the per-file `.1st-pass.model.json` sidecar (`FirstPassModelIO`) - not on the live process environment, because a diff --git a/pwiz_tools/Osprey/docs/12-second-pass-fdr.md b/pwiz_tools/Osprey/docs/12-second-pass-fdr.md index c0c23d6c09..70f5546aee 100644 --- a/pwiz_tools/Osprey/docs/12-second-pass-fdr.md +++ b/pwiz_tools/Osprey/docs/12-second-pass-fdr.md @@ -28,8 +28,8 @@ That is why the `percolator` mode was **removed** rather than demoted. It was measured at 1.57% true FDP against a nominal 1% on Stellar libdecoy entrapment (the first-pass q gives 0.92% on the same data), and around 9% on an 82-file SEA-AD set — the error grows with run count. **The linear model trained by the -first-pass SVM is now the model for the second pass in every mode**; only the -`OSPREY_PROTEIN_COMPACT_RETRAIN` diagnostic A/B still retrains. +first-pass SVM is now the model for the second pass in every mode**, and second-pass +retraining has been removed outright - see "Frozen vs. retrain" below. ## `OSPREY_PASS2_QVALUE` modes @@ -43,12 +43,31 @@ fails in seconds rather than after Stage 1-5. |------|----------|-----------------|-------| | `protein-compact` (default) | no (frozen model) | competition constrained to the protein stratum | precursor | | `transfer` | no | pass-1 q carried through; only moved peaks re-mapped | precursor + peptide | -| `transfer-compete` | no (frozen model) | fresh full-population target-decoy competition | precursor | + +These two are the whole list. `percolator` was removed for retraining against a +decoy-depleted null (above), and the `OSPREY_PROTEIN_COMPACT_RETRAIN` A/B toggle went with +the retrain it existed to measure. + +`transfer-compete` was removed for a **related but distinct** reason, and it is worth +stating precisely because the surviving default shares part of it. Its competition ran over +a **target-conditioned subset**: survivors were selected by per-run q on the TARGET side, +and decoys entered only by `base_id` pairing with those targets. That strips decoys which +WON the first-pass competition - the highest-scoring part of the null - so the second-pass q +improves with **no added evidence**. Measured on the 82-file SEA-AD cohort: **1.96% true FDP +at a nominal 1%, accepting 34,325**, against **1.53% and 37,624** for `protein-compact`. It +is dominated on both axes, which is why it is a removal rather than a demotion. + +`protein-compact` has the same paired-subsetting bias - its stratum gate is target-conditioned +too, tracked as **[#4581](https://github.com/ProteoWizard/pwiz/issues/4581)** (open), with +[#4560](https://github.com/ProteoWizard/pwiz/issues/4560) on the mixed-pass statistics that +ride along. The difference is that `protein-compact` also brings genuinely new protein-level +evidence to the ranking, where `transfer-compete` brought none. See also issue #4484 (closed) +for the default decision, and #4363 (closed) for the depleted-null finding. **Interaction with `OSPREY_EXPERIMENT_AGG`**: after a first pass run under the -experimental `mean-best-` aggregation, `transfer-compete` and `protein-compact` are -**refused** - both would rewrite the reported experiment q from a MAX-aggregated -competition. `transfer` is the compatible mode. See +experimental `mean-best-` aggregation, `protein-compact` is **refused** - it would +rewrite the reported experiment q from a MAX-aggregated competition. `transfer` is the +compatible mode. See [Experiment-wide aggregation](07-fdr-control.md#experiment-wide-aggregation-osprey_experiment_agg). ### `transfer` @@ -60,19 +79,15 @@ bins + PAVA isotonic; `LookupQForScore`). The experiment q is frozen by the best-peak anchor. Each survivor is classified Unchanged / Moved / GapFill, with bit-exact score equality as the "Moved" discriminator. -### `transfer-compete` (frozen model) - -Scores the reconciled **targets and decoys** with the **frozen 1st-pass model** -(no retrain), then recomputes q-values and PEP by a **fresh full-population -target-decoy competition** — a non-depleted null, because both sides are scored on -the same frozen scale (`ComputePass2TransferCompeteFull` with `stratumBaseIds == -null` → `PercolatorFdr.ComputeFullPopulationPrecursorFdrStreaming`, one file -resident at a time). Precursor-level only. - ### `protein-compact` (frozen model) -Identical to `transfer-compete` but the competition is **constrained to the -protein stratum** — the `base_id`s of proteins that had ≥2 peptides pass first-pass +Scores the reconciled **targets and decoys** with the **frozen 1st-pass model** (no +retrain), then recomputes q-values and PEP by a fresh target-decoy competition +(`ComputePass2TransferCompeteFull`, one file resident at a time). Both sides are scored on +the same frozen scale, which is what removes the RETRAIN pathology - but note this does not +make the null unbiased: the stratum gate is target-conditioned, so the in-stratum decoy null +is selected against (#4581). The competition is +**constrained to the protein stratum** — the `base_id`s of proteins that had ≥2 peptides pass first-pass protein FDR, admitted as target+decoy pairs (the stratum is built by first-pass protein parsimony; see [08-protein-parsimony.md](08-protein-parsimony.md)). Off-stratum survivors keep their first-pass q-values, so the report is @@ -89,49 +104,48 @@ Bourgon 2010). through `PercolatorFdr.ScoreStandardizedRow`, so it is classifier-agnostic and works for `--fdr-method gbdt` too). The model is captured on the streaming first pass via the `captureModel` hook. -- **Retrain** trains a fresh SVM/GBDT on the post-reconciliation pool. Since the - `percolator` mode was removed, the `OSPREY_PROTEIN_COMPACT_RETRAIN` A/B toggle - is the ONLY way to reach it. +- **Retrain** trained a fresh SVM/GBDT on the post-reconciliation pool. **It is gone.** + `percolator` was removed for the depleted-null reason above, and the + `OSPREY_PROTEIN_COMPACT_RETRAIN` A/B toggle that was the last way to reach it has been + removed too - the question it measured is settled and recorded here. Do not re-add it; + git history holds the dropped approach. -`OSPREY_PROTEIN_COMPACT_RETRAIN` is a diagnostic A/B lever: with `protein-compact` -it **skips** the frozen-model + stratum competition and instead retrains the -second pass over the stratum-expanded compacted pool, isolating the -frozen-vs-retrain calibration difference for the entrapment oracle. +**There is therefore no second-pass model.** The linear model the first-pass SVM trained +IS the model for pass 2, unchanged. Only the score DISTRIBUTIONS differ, because pass 2 +runs on a subset - which is why a pass-2 feature-contribution view needs the frozen +coefficients plus per-feature target/decoy means, and nothing that has to be retrained. ## Inputs from the first pass The frozen modes are frozen against artifacts, not against in-process state, so a `SecondPassFDR` node that never ran the first pass reads everything it needs from -disk. Two experiment-wide artifacts carry it: +disk. Three experiment-wide artifacts carry it: | Artifact | Carries | |---|---| -| `.1st-pass.model.json` | the frozen Percolator model (standardizer + per-fold weights and biases), the first pass's `OSPREY_EXPERIMENT_AGG` provenance, and - under `protein-compact` - the protein stratum | +| `.1st-pass.model.json` | the frozen Percolator model (standardizer + per-fold weights and biases) and the first pass's `OSPREY_EXPERIMENT_AGG` provenance | +| `.1st-pass.stratum.json` | the protein stratum, under `protein-compact`. Split out of the model sidecar in #4633, because first-pass protein FDR computes it and training does not | | `.1st-pass.fdr_experiment.bin` | the first pass's experiment-scope q-values | -**They must relay together.** A node holding one without the other cannot proceed: +**They must relay together.** A node holding one without the others cannot proceed: the model without the stratum cannot constrain a `protein-compact` competition, and the stratum without the model has nothing to score with. The model sidecar is written beside **every** run's other Stage-5 artifacts, identical each time, so any one copy serves. -> **In flight** - the stratum is moving out of the model sidecar into its own -> `.1st-pass.stratum.json`, because first-pass protein FDR computes it and training does -> not. Relay obligations are unchanged in substance: the two still travel together, as -> two files rather than one. See -> `Skyline/work/20260901_osprey_firstpass_resume`. - See [00-pipeline-architecture.md](00-pipeline-architecture.md) for the full contract and the per-boundary relay checklist. ## Fail-fast -An **explicitly requested** frozen mode never silently degrades to the -anti-conservative retrain. If the frozen model, the required sidecars, or the -protein stratum are absent (e.g. a warm rerun that loaded cached SVM scores and -skipped first-pass training, or a present-but-corrupt first-pass sidecar), -`Pass2FdrSidecar` aborts with a `ConfigError` and actionable guidance rather than -reporting looser FDR than a cold run under the same flag. +An **explicitly requested** frozen mode never silently degrades. If the frozen model, +the required sidecars, or the protein stratum are absent (e.g. a warm rerun that loaded +cached SVM scores and skipped first-pass training, or a present-but-corrupt first-pass +sidecar), `Pass2FdrSidecar` aborts with a `ConfigError` and actionable guidance rather +than reporting looser FDR than a cold run under the same flag. + +There is no longer anything to degrade TO - the retrain that used to be the fallback is +gone - so the abort is the only outcome, not the stricter of two. Because `protein-compact` is now the DEFAULT, this fail-fast reaches ordinary runs, not just explicitly flagged ones. A distributed `--task SecondPassFDR` node @@ -161,15 +175,14 @@ would be harder to reason about than an explicit variable. | Flag / env var | Default | Effect | |---|---|---| -| `OSPREY_PASS2_QVALUE` | `protein-compact` | Selects the second-pass q-value mode: `transfer` \| `transfer-compete` \| `protein-compact`. Unrecognized → startup error. | -| `OSPREY_PROTEIN_COMPACT_RETRAIN` | off (frozen) | With `protein-compact`, retrain the second pass instead of using the frozen model + stratum competition (A/B lever). | +| `OSPREY_PASS2_QVALUE` | `protein-compact` | Selects the second-pass q-value mode: `transfer` \| `protein-compact`. Unrecognized → startup error. | | `OSPREY_FDR_PROJECTION` | on | Streams the FDR peak via the thin `FdrProjection` slice; the frozen modes stream one file at a time so routing them does not hold all features resident. | ## Divergences from the Rust documentation - **[C#-ORIGINATED] The pass-2 frozen q-value modes originated in C#** - The Rust algorithm doc set has no second-pass-FDR document because these modes - (`transfer`, `transfer-compete`, `protein-compact`, and the frozen-model + (`transfer`, `protein-compact`, and the frozen-model machinery) were developed in the C# implementation first; the Rust reference is porting them back in maccoss/osprey#57. Both implementations have since removed the `percolator` mode and defaulted to `protein-compact` together, so the shipped diff --git a/pwiz_tools/Osprey/docs/20-command-line.md b/pwiz_tools/Osprey/docs/20-command-line.md index 792846b055..f992eaea8c 100644 --- a/pwiz_tools/Osprey/docs/20-command-line.md +++ b/pwiz_tools/Osprey/docs/20-command-line.md @@ -215,7 +215,7 @@ CLI; they are read once at process start. The ones most likely to matter: | `OSPREY_PICK_DUMP_CANDIDATES` | Dump per-candidate pick terms for offline model training | [peak-model-training.md](peak-model-training.md) | | `OSPREY_TRAIN_PICK_RUN` | First-pass training selection, **on by default**: each precursor is represented by one uniformly drawn run's best candidate peak. `OSPREY_TRAIN_PICK_RUN=0` restores the pre-26.1 cross-run maximum. C#-only — Rust still takes the maximum | [07](07-fdr-control.md) | | `OSPREY_MAX_TRAIN_SIZE` | Cap on training rows (default 300000). Unchanged by the 26.1 selection flip: at matched FDP, 300K and 1M are indistinguishable | [07](07-fdr-control.md) | -| `OSPREY_PASS2_QVALUE` | Second-pass q-value mode: `protein-compact` (**default**) / `transfer-compete` / `transfer`. An unrecognized value is a startup ERROR - `percolator` was removed | [12](12-second-pass-fdr.md) | +| `OSPREY_PASS2_QVALUE` | Second-pass q-value mode: `protein-compact` (**default**) / `transfer`. An unrecognized value is a startup ERROR - `percolator` and `transfer-compete` were removed | [12](12-second-pass-fdr.md) | | `OSPREY_GBT_*` | GBDT hyperparameters (with `--fdr-method gbdt`) | [07](07-fdr-control.md) | | `OSPREY_EXPERIMENT_AGG` | Experimental first-pass experiment-wide aggregation (`max` / `mean-best-`) | [07](07-fdr-control.md) | | `OSPREY_MEANBEST2_FLOOR_MEAN` / `OSPREY_MEANBEST2_FLOOR_PCT` | Missing-run floor arm for `mean-best-` (decoy mean / decoy percentile instead of the default median) | [07](07-fdr-control.md) | diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 6d3263c9d0..a9851ecce2 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -1549,7 +1549,7 @@ function Invoke-HpcChain { Copy-Item (Join-Path $ph3 "$s.1st-pass.fdr_scores.bin") (Join-Path $ph3Out "$s.1st-pass.fdr_scores.bin") # WITHHELD ONLY WHEN THE WORKER ANSWERED. The modes with a per-file half - # (protein-compact, transfer-compete) leave a 2nd-pass sidecar here, and phase 4 folds it + # (protein-compact) leaves a 2nd-pass sidecar here, and phase 4 folds it # without opening anything from the first pass - that is the contract issue #4486 # establishes, and withholding is how it is proven. OSPREY_PASS2_QVALUE=transfer and the # retrain modes have NO per-file half, so Stage 7 legitimately recomputes and legitimately @@ -1562,7 +1562,7 @@ function Invoke-HpcChain { Copy-Item (Join-Path $ph3 "$s.calibration.json") (Join-Path $ph4 "$s.calibration.json") Copy-Item (Join-Path $ph3 "$s.reconciliation.json") (Join-Path $ph4 "$s.reconciliation.json") # Ship the persisted 1st-pass model so SecondPassFDR can run the frozen 2nd-pass - # modes (transfer / transfer-compete / protein-compact) without re-training. Written + # modes (transfer / protein-compact) without re-training. Written # by the FirstPassFDR join node (phase 2) and relayed into $ph3 above ($ph2 is already # deleted by now). Present for the SVM/percolator framework, so guard with Test-Path. # protein-compact's stratum is its own artifact (protein FDR computes it, training does From d969570a3cc761c8beceb0278e87ee20358d782b Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Sun, 6 Sep 2026 20:01:58 -0700 Subject: [PATCH 05/30] Extracted transfer's per-run body so its caller can move to the fan-out * Lifted TransferPerRunQ's loop body out as TransferOneFile, unchanged. It builds that run's own score->q tables from that run's own .1st-pass.fdr_scores.bin and classifies that run's survivors; the only state crossing runs is the analysis-wide experiment map every run shares, and a handful of counters. That is a fan-out computation running in a join, which is the thing to fix - #4438 already made the algorithm per-run, only its home is wrong * Replaced six counter locals with a TransferTally struct, so the extraction did not need six ref parameters to be worth doing Behaviour is unchanged, including one place it nearly was not: the whole-run loop counted a run as done at the END of its body, AFTER the continue for an unreadable 1st-pass sidecar, so such a run was never counted. Incrementing at the call site instead would have quietly started counting it. The increment sits inside the method, where the loop had it. The mode still runs in SecondPassFDR; only the seam exists. Moving the call into Pass2PerFileWorker is next. NOT COVERED BY THE GATE, and this is worth knowing before reading the green above: regression.ps1 never sets OSPREY_PASS2_QVALUE, so all 15 legs run the default and no leg exercises transfer at all. What is covered is the algorithmic core this extraction reuses untouched - BuildScoreToQTable and AssignPerRunQ both have unit tests. The end-to-end oracle is the 82-file SEA-AD arm, run last; the numbers it must reproduce are in the TODO. The codebase has already paid for this gap once: transfer silently wrote no experiment sidecar at all until 2026-09, and nothing caught it. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 173 +++++++++++------- 1 file changed, 103 insertions(+), 70 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index 6564bac2ef..fad20a609b 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -1,4 +1,4 @@ -/* +/* * Original author: Brendan MacLean , * MacCoss Lab, Department of Genome Sciences, UW * AI assistance: Claude Code (Claude Opus 4.8) @@ -2953,7 +2953,7 @@ internal static bool TransferPerRunQ( } var scratch = new double[nFeatures]; // reused per entry to avoid a per-row allocation - int nUnchanged = 0, nMoved = 0, nGapFill = 0, nSkipped = 0, nMissingSidecar = 0, nFilesDone = 0; + var tally = new TransferTally(); // Per-file progress: building each file's per-run tables + classifying its survivors ran // silently for minutes on an 82-file join (the gap between Stage 6 and the summary below). var transferProgress = new ProgressReporter( @@ -2965,71 +2965,11 @@ internal static bool TransferPerRunQ( transferProgress.Report(++transferIdx); if (!inputByFileName.TryGetValue(kvp.Key, out string inputFile)) { - nSkipped += kvp.Value.Count; + tally.Skipped += kvp.Value.Count; continue; } - string pass1Path = FdrScoresSidecar.Pass1Path(inputFile); - - // Build this file's per-run tables + record map from its own 1st-pass sidecar. - var firstPassByEntryId = new Dictionary(); - var precScores = new List(); - var precQs = new List(); - var pepScores = new List(); - var pepQs = new List(); - bool ok = FdrScoresSidecar.ReadRecords( - pass1Path, FdrScoresSidecar.Pass.FirstPass, rec => - { - firstPassByEntryId[rec.EntryId] = rec; // entry_id is unique per file (DeduplicatePairs) - precScores.Add(rec.Score); - precQs.Add(rec.RunPrecursorQvalue); - pepScores.Add(rec.Score); - pepQs.Add(rec.RunPeptideQvalue); - }); - if (!ok || precScores.Count == 0) - { - nMissingSidecar++; - ctx.LogWarning(string.Format( - "OSPREY_PASS2_QVALUE=transfer: could not read the 1st-pass sidecar for '{0}' " + - "({1}); this file's per-run q is left unadjusted.", kvp.Key, pass1Path)); - continue; - } - BuildScoreToQTable(precScores, precQs, out double[] precScoresDesc, out double[] precQDesc); - BuildScoreToQTable(pepScores, pepQs, out double[] pepScoresDesc, out double[] pepQDesc); - - foreach (var entry in kvp.Value) - { - if (entry.Features == null || entry.Features.Length != nFeatures) - { - // No reconciled features resolved (a stub/parquet mismatch the reload - // already warned about). Leave this entry's q as-is rather than guess. - nSkipped++; - continue; - } - double newScore = ScoreWithFrozenModel( - entry.Features, standardizer, avgWeights, avgBias, scratch); - - FdrScoreRecord? rec1 = null; - if (firstPassByEntryId.TryGetValue(entry.EntryId, out FdrScoreRecord recFound)) - rec1 = recFound; - // The precursor's analysis-wide pass-1 experiment record, which supplies - // every disposition: an UNCHANGED or MOVED peak carries these values through, - // and a gap-fill peak (no 1st-pass run-scope record) takes them so - // ClampExperimentQToBestRun - a floor that only raises - lands it at the - // precursor's best-run q. A precursor with no record anywhere gets the - // default 1.0 q-values and a 0.0 aggregate, which pair correctly: never - // competed, never accepted, so nothing reads it. - FdrExperimentRecord? exp1 = null; - if (globalExperiment.TryGetValue(entry.EntryId, out var expFound)) - exp1 = expFound; - switch (AssignPerRunQ(entry, newScore, rec1, exp1, - precScoresDesc, precQDesc, pepScoresDesc, pepQDesc)) - { - case PerRunClass.Unchanged: nUnchanged++; break; - case PerRunClass.Moved: nMoved++; break; - default: nGapFill++; break; - } - } - nFilesDone++; + TransferOneFile(kvp.Key, inputFile, kvp.Value, standardizer, avgWeights, + avgBias, nFeatures, scratch, globalExperiment, ctx.LogWarning, ref tally); } transferProgress.Dispose(); @@ -3037,12 +2977,12 @@ internal static bool TransferPerRunQ( "OSPREY_PASS2_QVALUE=transfer: per-run q transfer over {0} file(s) -- {1} unchanged " + "(pass-1 q carried), {2} moved (run q re-mapped, experiment q carried), {3} gap-fill " + "(new run q + carried experiment q){4}{5}.", - nFilesDone, nUnchanged, nMoved, nGapFill, - nMissingSidecar > 0 - ? string.Format("; {0} file(s) had no readable 1st-pass sidecar", nMissingSidecar) + tally.FilesDone, tally.Unchanged, tally.Moved, tally.GapFill, + tally.MissingSidecar > 0 + ? string.Format("; {0} file(s) had no readable 1st-pass sidecar", tally.MissingSidecar) : string.Empty, - nSkipped > 0 - ? string.Format("; {0} entr(y/ies) skipped for missing features", nSkipped) + tally.Skipped > 0 + ? string.Format("; {0} entr(y/ies) skipped for missing features", tally.Skipped) : string.Empty)); // PUBLISH THE EXPERIMENT SCOPE, exactly as the competition modes do. This mode writes @@ -3061,6 +3001,99 @@ internal static bool TransferPerRunQ( return true; } + /// Counts one call adds to. A struct rather + /// than six ref parameters, so the per-file body could be lifted out of the whole-run + /// loop without its signature becoming the reason not to. + internal struct TransferTally + { + public int Unchanged, Moved, GapFill, Skipped, MissingSidecar, FilesDone; + } + + /// + /// Transfer ONE run's per-run q-values: build that run's own score->q tables from its own + /// .1st-pass.fdr_scores.bin, then classify and re-map its survivors. + /// + /// The body of 's loop, extracted unchanged. Nothing + /// in it reads another run's state - the tables come from this run's sidecar and the + /// experiment records are the analysis-wide map every run shares - which is what makes + /// the mode a fan-out computation that happens to be running in the join. Moving the + /// CALLER is the point; this seam is what lets that happen without rewriting the + /// algorithm (#4438 established the per-run form; only its home is still wrong). + /// + internal static void TransferOneFile( + string fileName, string inputFile, List survivors, + FeatureStandardizer standardizer, double[] avgWeights, double avgBias, + int nFeatures, double[] scratch, + IReadOnlyDictionary globalExperiment, + Action logWarning, ref TransferTally tally) + { + string pass1Path = FdrScoresSidecar.Pass1Path(inputFile); + + // Build this file's per-run tables + record map from its own 1st-pass sidecar. + var firstPassByEntryId = new Dictionary(); + var precScores = new List(); + var precQs = new List(); + var pepScores = new List(); + var pepQs = new List(); + bool ok = FdrScoresSidecar.ReadRecords( + pass1Path, FdrScoresSidecar.Pass.FirstPass, rec => + { + firstPassByEntryId[rec.EntryId] = rec; // entry_id is unique per file (DeduplicatePairs) + precScores.Add(rec.Score); + precQs.Add(rec.RunPrecursorQvalue); + pepScores.Add(rec.Score); + pepQs.Add(rec.RunPeptideQvalue); + }); + if (!ok || precScores.Count == 0) + { + tally.MissingSidecar++; + logWarning(string.Format( + "OSPREY_PASS2_QVALUE=transfer: could not read the 1st-pass sidecar for '{0}' " + + "({1}); this file's per-run q is left unadjusted.", fileName, pass1Path)); + return; + } + BuildScoreToQTable(precScores, precQs, out double[] precScoresDesc, out double[] precQDesc); + BuildScoreToQTable(pepScores, pepQs, out double[] pepScoresDesc, out double[] pepQDesc); + + foreach (var entry in survivors) + { + if (entry.Features == null || entry.Features.Length != nFeatures) + { + // No reconciled features resolved (a stub/parquet mismatch the reload + // already warned about). Leave this entry's q as-is rather than guess. + tally.Skipped++; + return; + } + double newScore = ScoreWithFrozenModel( + entry.Features, standardizer, avgWeights, avgBias, scratch); + + FdrScoreRecord? rec1 = null; + if (firstPassByEntryId.TryGetValue(entry.EntryId, out FdrScoreRecord recFound)) + rec1 = recFound; + // The precursor's analysis-wide pass-1 experiment record, which supplies + // every disposition: an UNCHANGED or MOVED peak carries these values through, + // and a gap-fill peak (no 1st-pass run-scope record) takes them so + // ClampExperimentQToBestRun - a floor that only raises - lands it at the + // precursor's best-run q. A precursor with no record anywhere gets the + // default 1.0 q-values and a 0.0 aggregate, which pair correctly: never + // competed, never accepted, so nothing reads it. + FdrExperimentRecord? exp1 = null; + if (globalExperiment.TryGetValue(entry.EntryId, out var expFound)) + exp1 = expFound; + switch (AssignPerRunQ(entry, newScore, rec1, exp1, + precScoresDesc, precQDesc, pepScoresDesc, pepQDesc)) + { + case PerRunClass.Unchanged: tally.Unchanged++; break; + case PerRunClass.Moved: tally.Moved++; break; + default: tally.GapFill++; break; + } + } + // Counted only HERE, where the whole-run loop counted it: the unreadable-sidecar + // path above returns first, so a run whose sidecar could not be read was never one + // this pass finished. + tally.FilesDone++; + } + /// /// Collapse the per-file survivors into the one-record-per-entry_id experiment scope. /// From 73dda2efaa938ff392aa6579362bc3e5794ddbba Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Sun, 6 Sep 2026 20:55:07 -0700 Subject: [PATCH 06/30] Added a gate leg for the two non-default pass-2 arms * Added mode 10: transfer, and mean-best-2 + transfer, each run straight through and asserted to PRODUCE - output.blib, the analysis-wide output.2nd-pass.fdr_experiment.bin, and a per-run .2nd-pass.fdr_scores.bin for every input. Neither arm had ever run under the gate: every OSPREY_PASS2_QVALUE mention in regression.ps1 was a comment, so all legs ran the default. The cost of that is already recorded in the code - transfer reached production writing no experiment sidecar at all, and nothing could see it * Asserted the artifact contract rather than values, on purpose. These two arms are still moving - protein-compact has improvements pending and the 82-file comparison needs re-running against #4593 - so a golden would freeze a number nobody has agreed on. What must not change silently is that a supported mode still runs and still produces * Opted in ONE dataset, via AltPass2 on StellarLibDecoy. A leg written into the per-dataset loop inherits a 4x multiplier - four configs, not two acquisitions - so this would have added eight straight-through runs and wrecked the 1:05 that -Dataset All was just tuned to. StellarLibDecoy because library-SUPPLIED decoys are what the pass-2 comparison runs on real cohorts, so the arms meet the decoy provenance they are used with, at Stellar speed and off Astral's critical path * Added -SkipAltPass2 for local iteration, alongside the existing -Skip* switches Measured: 253.3s and 255.0s, so 8.5 min for both arms on StellarLibDecoy. That exceeds the ~5 min the second lane finishes early by, so under -Dataset All two arms move lane 2 onto the critical path. One arm fits free, and the mean-best-2 arm runs transfer for pass 2 anyway (protein-compact refuses a mean(best-N) first pass), so one leg does exercise both ideas. regression.ps1 -Dataset StellarLibDecoy: PASSED, 23 legs, both new arms green on first run. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- pwiz_tools/Osprey/regression.ps1 | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index a9851ecce2..dac9e613d8 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -183,6 +183,10 @@ param( [switch]$SkipWarmRerun, [switch]$SkipRehydrate, [switch]$SkipHpcChain, + # Skip the mode-10 non-default pass-2 arms (transfer, mean-best-N). Fast local iteration + # only: these are the two ideas in competition with the shipped default, and leaving them + # unrun is how transfer reached production writing no experiment sidecar. + [switch]$SkipAltPass2, [string]$DownloadsPath, [int]$Threads = 16, [switch]$TeamCity, @@ -386,8 +390,19 @@ $libDecoyV3Url = 'https://panoramaweb.org/_webdav/MacCoss/software/%40files/perf # (default 0.05), same do-not-regenerate rule. Only bites on a # dataset that carries entrapment. $datasets = [ordered]@{ + # AltPass2 opts ONE dataset into mode 10 (the non-default pass-2 arms), and picking one + # is the point. A new leg applied to every config inherits a 4x multiplier - four configs, + # not two acquisitions - which multiplies wall time without multiplying coverage. Choose the + # config the case actually belongs to. + # + # For these arms that is StellarLibDecoy: library-SUPPLIED decoys are what the pass-2 + # comparison runs on real cohorts (SEA-AD is -DecoyMode libdecoy), so the arms get exercised + # against the decoy provenance they are actually used with, at Stellar speed. Not Astral, + # which is the suite's critical path and pays for an extra straight-through run in wall + # clock directly. Stellar = @{ Folder = 'stellar'; Resolution = 'unit' } StellarLibDecoy = @{ + AltPass2 = $true Folder = 'stellar' LibraryFolder = 'stellar-libdecoy' GoldenFolder = 'stellar-libdecoy' @@ -2058,6 +2073,70 @@ foreach ($name in $selected) { } } + # ---- mode 10: the non-default pass-2 arms actually run ---- + # transfer and mean-best-N are the two ideas in competition with the shipped default, and + # until now NOTHING here exercised either: every leg above runs with OSPREY_PASS2_QVALUE + # unset. The cost of that has already been paid once - `transfer` silently wrote no + # analysis-wide 2nd-pass experiment sidecar at all, while every other mode wrote one, and + # no leg could see it because the arm had never run under the gate. + # + # So this leg asserts the ARTIFACT CONTRACT rather than values: the arm completes, and it + # leaves the same set of files behind that the default does. That is deliberately not a + # golden - these arms are still moving (protein-compact has improvements pending, and the + # 82-file comparison wants re-running), and a golden would freeze a number nobody has + # agreed on yet. What must not change silently is that the arm RUNS and PRODUCES. + # + # The two arms pair as they must: protein-compact REFUSES a mean(best-N) first pass, so the + # mean-best leg necessarily runs transfer as its pass-2 mode. + if (-not $SkipAltPass2 -and $cfg.AltPass2) { + $altArms = @( + @{ Tag = 'transfer'; Env = @{ OSPREY_PASS2_QVALUE = 'transfer' } }, + @{ Tag = 'meanbest2'; Env = @{ OSPREY_PASS2_QVALUE = 'transfer' + OSPREY_EXPERIMENT_AGG = 'mean-best-2' } }) + foreach ($arm in $altArms) { + Write-Progress-Tc "${name}: $($arm.Tag) arm runs and produces (mode 10)" + $altDir = Join-Path (Join-Path $runRoot $name) ("alt-" + $arm.Tag) + $m10 = [pscustomobject]@{ Issues = [System.Collections.Generic.List[string]]::new() } + foreach ($k in $arm.Env.Keys) { Set-Item -Path "Env:$k" -Value $arm.Env[$k] } + try { + $rAlt = Invoke-OspreyRun -Mzmls $inputs.Mzmls -Library $inputs.Library ` + -Resolution $cfg.Resolution -WorkDir $altDir -LogName "alt-$($arm.Tag).log" ` + -Spec $cfg -Manifest $inputs.Manifest + } finally { + foreach ($k in $arm.Env.Keys) { Remove-Item -Path "Env:$k" -ErrorAction SilentlyContinue } + } + $altBlib = Join-Path $altDir 'output.blib' + if (-not (Test-Path $altBlib) -or (Get-Item $altBlib).Length -eq 0) { + $m10.Issues.Add("$($arm.Tag): no output.blib written") + } + # The analysis-wide 2nd-pass experiment sidecar - the artifact whose absence went + # unnoticed. Named for the blib stem, one per analysis. + $altExp = Join-Path $altDir 'output.2nd-pass.fdr_experiment.bin' + if (-not (Test-Path $altExp)) { + $m10.Issues.Add(("$($arm.Tag): no output.2nd-pass.fdr_experiment.bin - the arm " + + "completed without writing the experiment-scope sidecar every " + + "other mode writes")) + } + # And one per-run 2nd-pass sidecar per input, the per-run half of the same contract. + foreach ($mz in $inputs.Mzmls) { + $stem = [IO.Path]::GetFileNameWithoutExtension($mz) + if (-not (Test-Path (Join-Path $altDir "$stem.2nd-pass.fdr_scores.bin"))) { + $m10.Issues.Add("$($arm.Tag): no 2nd-pass FDR sidecar for $stem") + } + } + if ($m10.Issues.Count -eq 0) { + $summaryLines.Add("$name mode10 ($($arm.Tag) arm runs and produces): PASS") + } else { + $overallFail = $true + Write-Problem-Tc ("$name mode10 ($($arm.Tag) arm): FAIL -- " + + "$($m10.Issues.Count) issue(s)") + $summaryLines.Add(("$name mode10 ($($arm.Tag) arm runs and produces): FAIL " + + "($($m10.Issues.Count) issues)")) + foreach ($iss in $m10.Issues) { Write-Host " $iss" -ForegroundColor Red } + } + } + } + # ---- mode 4: warm re-run cache-hit assertion ---- # The gate's structural blind spot, closed. Modes 1 and 3 compare output produced # in FRESH dirs, and mode 2 invalidates Stage 5 before it re-runs, so no leg here From 4d0bae8caf9184d23a3d79272f049520fb6049b5 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Sun, 6 Sep 2026 22:44:13 -0700 Subject: [PATCH 07/30] Made SecondPassFDR fold over the runs instead of holding them * Root cause corrected first. The 446-file wall is NOT the Stage 6 -> 7 pool rebuild the earlier measurement was attributed to: on the --task SecondPassFDR leg the run dies inside `--input-scores: loading 446 per-file score parquet(s)`, at run 381 of 446, before Stage 7 computes anything. The merge materializes every run's post-compaction survivors up front, and THAT is the 68.0 GB. BuildRescoredPool is never reached * So the fix is at the hydrate, not at the pool build. CanStreamStage7Join admits the reconciled-input leg to the same per-run shape CanHydratePerRun already gives the rescore: the load publishes one EMPTY list per run and reads no rows, and Stage 7 rebuilds a run from its own .scores-reconciled.parquet + 1st-pass sidecar, folds it, and drops it * RescoreHydration.RefillOneRunSurvivors is that rebuild - the three steps HydrateOneRun takes (load, overlay, compact) and not the fourth. A join runs no rescore, so reading each run's reconciliation.json to plan actions nobody executes would push ~6 MB of join-wide first_pass_base_ids through the call 446 times * Every Stage 7 consumer now folds through RescoredEntries.StreamFiles: the fragment release, the pass-2 competition, protein FDR, the experiment-q re-clamp, all three blib gates and the FDRBench writer. Each was already a fold to O(distinct); none of them ever needed the pool, which is why the conversion is a change of source and not of algorithm * The two facts a resident pool carries BETWEEN passes - the 2nd-pass sidecar overlay and the experiment-q floors - are re-applied per run instead, in that order, through AddPostMaterialize. A floor raises a value the overlay has just written, so the composition order is the correctness argument * ClampExperimentQToBestRun is now fold-then-apply over the stream, which is the shape PercolatorEngine already split its two halves for OSPREY_STAGE7_STREAM=0 keeps the resident pool as the byte-identity oracle, and carries a validity-key term so an in-place A/B cannot satisfy itself by adopting the other arm's .blib. NOT converted, and named rather than left to be discovered: --model-diagnostics. Its pass-2 builders index their files by position and revisit a file across two loops, so they need a list. CanStreamStage7Join declines that leg outright rather than letting it stream and then pull the whole pool back through .Value - the same peak by a longer route, with nothing in the log to say so. The fold it wants is the accumulator the pass-1 report already uses. Also fixed a defect this branch introduced: TransferOneFile's missing-features branch was a `return` where the loop it was extracted from had a `continue`, so one entry without reconciled features abandoned the rest of that run's survivors AND skipped the run's FilesDone count. 605 tests (604 pass, 1 pre-existing skip), inspection zero-warning. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Core/OspreyEnvironment.cs | 33 ++++ .../Osprey.Tasks/FdrBenchInputWriter.cs | 2 +- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 179 +++++++++++++----- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 107 ++++++++++- .../Osprey/Osprey.Tasks/PerFileScoringTask.cs | 18 +- .../Osprey/Osprey.Tasks/PipelineByproducts.cs | 111 +++++++++++ .../Osprey/Osprey.Tasks/RescoreHydration.cs | 57 ++++++ .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 44 ++++- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 87 +++++++-- 9 files changed, 575 insertions(+), 63 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs b/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs index 5d7d17cee5..c388e629fb 100644 --- a/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs +++ b/pwiz_tools/Osprey/Osprey.Core/OspreyEnvironment.cs @@ -198,6 +198,27 @@ public static class OspreyEnvironment public static bool Stage6StreamSurvivors { get; set; } = IsNotZero(@"OSPREY_STAGE6_STREAM_SURVIVORS"); + /// + /// Stage 7 folds over the runs one at a time - rebuilding each run's survivors from its + /// own .scores-reconciled.parquet and 1st-pass sidecar, and dropping them again + /// once the fold has visited them - instead of being handed every run's survivors at + /// once. + /// + /// DEFAULT ON. The all-runs survivor pool is what a --task SecondPassFDR node + /// spends its whole memory budget on before the join computes anything: at 446 CHS runs + /// it reached 68.0 GB managed / 70.5 GB private and was killed at run 381 of 446 with + /// 0.34 GB free, still inside the --input-scores load. It is the + /// O(runs x entries) shape the architecture forbids a join to hold, and every + /// consumer of it in Stage 7 - the fragment release, the pass-2 competition, protein + /// FDR, the experiment-q re-clamp and all three blib gates - is a fold to + /// O(distinct) that never needed the whole pool. + /// + /// Set OSPREY_STAGE7_STREAM=0 to keep the resident pool as the A/B byte-identity oracle, + /// the same role OSPREY_STAGE6_STREAM_SURVIVORS=0 plays for the Stage 6 handoff. A + /// settable property (not a readonly field) so unit tests can A/B both paths. + /// + public static bool Stage7Stream { get; set; } = IsNotZero(@"OSPREY_STAGE7_STREAM"); + /// /// At the Stage 5 -> 6 boundary, drop LibraryEntry.Fragments for every library /// entry that can no longer be scored or written - i.e. everything outside the @@ -255,6 +276,18 @@ public static string Stage6StreamSurvivorsValidityKeySuffix() return Stage6StreamSurvivors ? string.Empty : @";stage6stream=0"; } + /// + /// Cache-validity suffix for the Stage 7 fold arm, on exactly the argument its Stage 6 + /// sibling above makes: empty on the streamed default so no existing output directory is + /// invalidated, and a term on the resident opt-out so an in-place A/B of the two arms + /// cannot satisfy itself by adopting the other arm's .blib and 2nd-pass sidecars + /// instead of recomputing them. + /// + public static string Stage7StreamValidityKeySuffix() + { + return Stage7Stream ? string.Empty : @";stage7stream=0"; + } + /// /// OSPREY_PICK_DUMP_CANDIDATES: when set to a non-empty / non-zero value, dump one /// row per CWT candidate peak of every precursor (targets AND decoys) scored in the diff --git a/pwiz_tools/Osprey/Osprey.Tasks/FdrBenchInputWriter.cs b/pwiz_tools/Osprey/Osprey.Tasks/FdrBenchInputWriter.cs index 1ea48262a4..998eae7a9d 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/FdrBenchInputWriter.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/FdrBenchInputWriter.cs @@ -116,7 +116,7 @@ public static string PathForPass(OspreyConfig config, int pass) /// Entrapment sequences to exclude (unmatched orphans); null to write all. public static Result WritePeptideInput( string path, - List>> perFileEntries, + IEnumerable>> perFileEntries, IReadOnlyDictionary libraryById, FdrLevel fdrLevel, bool perRun, diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index fad20a609b..270e92efc1 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -25,7 +25,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using pwiz.Osprey.Core; using pwiz.Osprey.FDR; using pwiz.Osprey.IO; @@ -87,10 +86,14 @@ internal static FeatureContributions ComputeAndPersist( string taskValidityKey) { var config = ctx.Config; - // The one buffer behind the milestone. Taken here rather than as a second - // parameter so the frozen-competition path (which reads the milestone) and - // every other path (which reads the buffer) cannot be handed different pools. - var perFileEntries = rescored.Value; + // The one buffer behind the milestone, and NOT read here (#4486). Taken through a + // local rather than as a second parameter so the frozen-competition path (which + // reads the milestone) and every other path (which reads the buffer) cannot be + // handed different pools - but read LAZILY, because on the frozen path nothing + // below asks for it and .Value is the O(runs x entries) build this stage exists to + // stop paying. The retrain-shaped paths (the resident 2nd pass, the projection + // sink's per-file protein-q map) genuinely index the whole pool and still get it. + List>> Pool() => rescored.Value; FeatureContributions pass2Contributions = null; // OSPREY_PASS2_QVALUE selects how this 2nd pass assigns reported q-values. @@ -212,7 +215,7 @@ internal static FeatureContributions ComputeAndPersist( // hide a name-drift bug that the standard cross-impl gate // (where keys always match) cannot catch. var unmatchedKeys = pass2Writer.UnmatchedKeys( - perFileEntries.Select(kvp => kvp.Key)); + rescored.FileNames); if (unmatchedKeys.Count > 0) { ctx.LogWarning(string.Format( @@ -225,14 +228,14 @@ internal static FeatureContributions ComputeAndPersist( int missingPass2 = 0; int totalFiles = 0; - foreach (var kvp in perFileEntries) + foreach (string fileKey in rescored.FileNames) { totalFiles++; // A key with no input file is not "missing" - it is unmatched, reported // above, and gets no sidecar either way. - if (pass2Writer.InputFor(kvp.Key) == null) + if (pass2Writer.InputFor(fileKey) == null) continue; - if (!pass2Writer.IsCurrent(kvp.Key)) + if (!pass2Writer.IsCurrent(fileKey)) missingPass2++; } // The RECOMPUTE gate, and only that. Every file gets a sidecar written below @@ -299,7 +302,7 @@ internal static FeatureContributions ComputeAndPersist( if (!frozenCompetition) { var swRestore = Stopwatch.StartNew(); - RestorePass1Scalars(ctx, perFileEntries, pass2Writer); + RestorePass1Scalars(ctx, Pool(), pass2Writer); swRestore.Stop(); ctx.LogVerbose(string.Format( "[STAGE-WALL] pass-1 scalar restore: {0:F1}s", swRestore.Elapsed.TotalSeconds)); @@ -345,7 +348,7 @@ internal static FeatureContributions ComputeAndPersist( // hence a experiment_protein_qvalue), so the last-write map is exact. var survivorsByFile = new Dictionary>(StringComparer.Ordinal); - foreach (var kvp in perFileEntries) + foreach (var kvp in Pool()) survivorsByFile[kvp.Key] = kvp.Value; IReadOnlyDictionary ResolveProteinQ(string fileName) @@ -370,7 +373,7 @@ void FlushPass2File(string fileName, IReadOnlyList records) } pass2Projections = ComputePass2Projection( - ctx, perFileEntries, perFileParquetPaths, config, + ctx, Pool(), perFileParquetPaths, config, ResolveProteinQ, FlushPass2File); } else @@ -378,7 +381,7 @@ void FlushPass2File(string fileName, IReadOnlyList records) // Resident 2nd pass (flag off): the byte-identity oracle. Reload // every survivor's PIN features resident, then run the resident // Percolator over the full FdrEntry survivor buffer. - pass2Contributions = ComputePass2Resident(ctx, perFileEntries, perFileParquetPaths, config); + pass2Contributions = ComputePass2Resident(ctx, Pool(), perFileParquetPaths, config); } swPass2.Stop(); ctx.LogInfo(string.Format( @@ -393,8 +396,8 @@ void FlushPass2File(string fileName, IReadOnlyList records) // back instead of quietly downgrading a resumed run's file to pass-1 values. A file // with no sidecar yet - a first run with no rescore work - simply has nothing to // load, and the write gives it the standing values, which are its answer. - if (!recomputed) - ReloadPass2Sidecars(ctx, pass2Writer, perFileEntries, @"pre-write"); + if (!recomputed && !rescored.Streams) + ReloadPass2Sidecars(ctx, pass2Writer, Pool(), @"pre-write"); // Persist post-Stage-6 per-file 2nd-pass FDR scores // BEFORE RunProteinFdr. The sidecar holds Score + @@ -423,7 +426,7 @@ void FlushPass2File(string fileName, IReadOnlyList records) // .2nd-pass sidecar written and the next resume re-runs // its second-pass FDR unnecessarily. var unmatchedSidecarKeys = pass2Writer.UnmatchedKeys( - perFileEntries.Select(kvp => kvp.Key)); + rescored.FileNames); if (unmatchedSidecarKeys.Count > 0) { ctx.LogWarning(string.Format( @@ -457,11 +460,11 @@ void FlushPass2File(string fileName, IReadOnlyList records) // the 38s gap perfviz reports between the competition's [STAGE-WALL] line // and the next probe (#4486). IO-paced, like the other disk loops here. using (var writeProgress = new ProgressReporter( - string.Format(@"Writing 2nd-pass FDR scores for {0} file(s)", perFileEntries.Count), - perFileEntries.Count, string.Empty, ProgressReporter.IO_INTERVAL_SECONDS)) + string.Format(@"Writing 2nd-pass FDR scores for {0} file(s)", rescored.FileCount), + rescored.FileCount, string.Empty, ProgressReporter.IO_INTERVAL_SECONDS)) { long nWrittenReported = 0; - foreach (var kvp in perFileEntries) + foreach (var kvp in Pool()) { writeProgress.Report(++nWrittenReported); pass2Writer.Write(kvp.Key, kvp.Value); @@ -501,9 +504,19 @@ void FlushPass2File(string fileName, IReadOnlyList records) // above already overlaid every sidecar onto these same entries and the // write put identical bytes back, so a second read of the whole sidecar // set (~4.8 GB at 82 files) applied values the entries already hold. - if (recomputed && perFileParquetPaths.Count > 0 && config.InputFiles != null) - ReloadPass2Sidecars(ctx, pass2Writer, perFileEntries, @"post-write"); - + // On the STREAMED pool neither reload runs, and the reason is that neither can: the + // entries they would overlay are dropped as each run is folded, so a pass over the + // whole cohort here would apply the sidecars to rows nothing reads and then throw + // them away. The same overlay is installed as a per-run hook instead + // (InstallStreamedPass2Overlay, called by the stage right after this method), so + // every later fold rebuilds a run and immediately gets its second-pass values. + // Same operation, same rows; applied when the row exists rather than in a pass of + // its own. + if (recomputed && !rescored.Streams && + perFileParquetPaths.Count > 0 && config.InputFiles != null) + { + ReloadPass2Sidecars(ctx, pass2Writer, Pool(), @"post-write"); + } return pass2Contributions; } @@ -575,31 +588,14 @@ private static void ReloadPass2Sidecars( foreach (var kvp in perFileEntries) { reloadProgress.Report(++nReloadReported); - string inputFile = writer.InputFor(kvp.Key); - if (inputFile == null) - continue; - string pass2Path = FdrScoresSidecar.Pass2Path(inputFile); - if (!FdrScoresSidecar.IsCurrentFormat(pass2Path, FdrScoresSidecar.Pass.SecondPass)) - { - filesMissing++; - continue; - } - var byEntryId = new Dictionary(kvp.Value.Count); - foreach (var e in kvp.Value) - byEntryId[e.EntryId] = e; - if (FdrScoresSidecar.TryReadOverlay( - pass2Path, byEntryId, FdrScoresSidecar.Pass.SecondPass, - experimentRecords)) + if (OverlayPass2SidecarOntoFile( + writer, kvp.Key, kvp.Value, experimentRecords, ctx.LogWarning)) { filesReloaded++; } else { filesMissing++; - ctx.LogWarning(string.Format( - "Failed to reload 2nd-pass FDR sidecar for {0} ({1}); " + - "protein FDR will use stale 1st-pass q-values", - kvp.Key, pass2Path)); } } } @@ -611,6 +607,77 @@ private static void ReloadPass2Sidecars( } } + /// + /// Overlay ONE run's .2nd-pass.fdr_scores.bin - plus the analysis-wide 2nd-pass + /// experiment records - onto that run's entries. The body of + /// ' loop, extracted because a STREAMED Stage 7 has to + /// apply it once per run per pass rather than once per run in total: the entries are + /// rebuilt from disk carrying their 1st-pass values, so without this every fold after + /// the second pass would read pass-1 q-values off pass-2 rows. + /// + /// Returns false when the run has no readable current sidecar - the caller decides + /// whether that is the legitimate absence (a first run with no rescore work) or the + /// failed write it is on the post-write pass, which is why the disposition is not taken + /// here. + /// + /// A missing sidecar leaves the entries EXACTLY as they arrived rather than + /// resetting them. On the resident path that is the standing first-pass state, which is + /// what the loop's own contract says such a run keeps; on the streamed path it is the + /// same state, freshly rebuilt. The two agree because neither invents a value. + /// + private static bool OverlayPass2SidecarOntoFile( + Pass2SidecarWriter writer, string fileName, List entries, + IReadOnlyDictionary experimentRecords, + Action logWarning) + { + string inputFile = writer.InputFor(fileName); + if (inputFile == null) + return false; + string pass2Path = FdrScoresSidecar.Pass2Path(inputFile); + if (!FdrScoresSidecar.IsCurrentFormat(pass2Path, FdrScoresSidecar.Pass.SecondPass)) + return false; + var byEntryId = new Dictionary(entries.Count); + foreach (var e in entries) + byEntryId[e.EntryId] = e; + if (FdrScoresSidecar.TryReadOverlay( + pass2Path, byEntryId, FdrScoresSidecar.Pass.SecondPass, experimentRecords)) + { + return true; + } + logWarning(string.Format( + "Failed to reload 2nd-pass FDR sidecar for {0} ({1}); " + + "protein FDR will use stale 1st-pass q-values", fileName, pass2Path)); + return false; + } + + /// + /// Make every fold that runs AFTER the second pass see the second pass's answer, on a + /// Stage 7 whose runs are rebuilt from disk one at a time (#4486). + /// + /// On the resident pool stamps the entries once + /// and every later pass reads the stamps. A streamed pool has no entries to stamp + /// between passes, so the same overlay is installed as a per-run hook and re-applied to + /// each run as it is rebuilt. Same operation, same rows, same result - once per run per + /// pass instead of once per run, which is the price of not holding the pool. + /// + /// The experiment records are resolved PER CALL rather than captured once, + /// deliberately: answers from the in-memory + /// accumulator once a pass-2 path has published one and from the on-disk sidecar + /// otherwise, and Stage 7 crosses that boundary partway through - protein FDR writes the + /// sidecar. Capturing the earlier answer would freeze the pre-competition values into + /// every later fold. + /// + internal static void InstallStreamedPass2Overlay( + PipelineContext ctx, RescoredEntries rescored, string taskName, string taskValidityKey) + { + if (!rescored.Streams) + return; + var writer = new Pass2SidecarWriter(ctx, ctx.Config, taskName, taskValidityKey); + rescored.AddPostMaterialize((fileName, entries) => + OverlayPass2SidecarOntoFile( + writer, fileName, entries, ResolvePass2ExperimentRecords(ctx), ctx.LogWarning)); + } + /// /// Re-seed each survivor's and /// from that file's .1st-pass.fdr_scores.bin. @@ -1621,10 +1688,16 @@ private static bool ComputePass2TransferCompeteFull( fileNames.Count, string.Empty, ProgressReporter.IO_INTERVAL_SECONDS)) { int mergeIdx = 0; - foreach (var kvp in rescored.Files()) + foreach (var kvp in rescored.StreamFiles()) { mergeProgress.Report(++mergeIdx); - residentByFile[kvp.Key] = kvp.Value; + // Only where the pool is going to stay. On a streamed source these lists are + // emptied the moment the fold moves on, so a map of references to them would + // hand the competition 446 empty runs - the failure that looks like a + // cohort with no survivors rather than like a bug. There LoadOneFile + // rebuilds the run it is asked for instead. + if (!rescored.Streams) + residentByFile[kvp.Key] = kvp.Value; survivorObservations += kvp.Value.Count; foreach (var e in kvp.Value) survivorEntryIds.Add(e.EntryId); @@ -1846,6 +1919,12 @@ private static bool ComputePass2TransferCompeteFull( // results forward. List LoadOneFile(string fileKey) { + // Rebuilt from this run's own artifacts on a streamed source, taken off the + // resident buffer when there is one. Either way it is ONE run, which is what + // this pass was already written to hold - the streamed source only makes + // that true of the stage around it as well. + if (rescored.Streams) + return rescored.MaterializeFile(fileKey); return residentByFile.TryGetValue(fileKey, out var resident) ? resident : new List(); @@ -1951,6 +2030,13 @@ void ApplyFileRunQ(string fileKey, StreamingFdr.FileCompetition contribution) sidecarsWritten.Add(fileKey); else if (!ctx.Config.DiagnosticsOnly) writeFailures.Add(fileKey); + // DROP the run here, on a streamed source: its answer is on disk and step 4 + // patches the sidecar rather than the entries, so this is the last line that + // reads them. Without it the pass would refill run after run and never let + // one go - the whole-run pool rebuilt one run at a time, which is the shape + // that looks like a fix in the code and like no fix at all in the profile. + if (rescored.Streams) + rescored.DropFile(fileKey); currentKey = null; currentEntries = null; } @@ -3060,9 +3146,12 @@ internal static void TransferOneFile( if (entry.Features == null || entry.Features.Length != nFeatures) { // No reconciled features resolved (a stub/parquet mismatch the reload - // already warned about). Leave this entry's q as-is rather than guess. + // already warned about). Leave this ENTRY's q as-is rather than guess, and + // go on to the next one - a `return` here would abandon the rest of the + // file's survivors at their Stage-6 q AND skip the FilesDone count below, + // which is a whole run silently dropped for one entry's missing features. tally.Skipped++; - return; + continue; } double newScore = ScoreWithFrozenModel( entry.Features, standardizer, avgWeights, avgBias, scratch); diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index 2a1ce42933..a8c0f4e91f 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -687,7 +687,26 @@ public override bool Rehydrate(PipelineContext ctx) // Publish the RescoredEntries milestone over the shared backing list // (the reconciled-input path applies its own compaction below, in place). - ctx.Publish(new RescoredEntries(_perFileEntries)); + // + // With a per-run source when this leg may stream (#4486): the --input-scores load + // then published one EMPTY list per run and read no rows, so the lists are filled + // by Stage 7's fold one run at a time and dropped again. The whole-run build is + // still supplied, and is what a consumer that reads .Value gets - it materializes + // every run through the same per-run source, so the two arms cannot disagree about + // what a run's post-compaction state is. It is the expensive answer, not a wrong + // one, which is the right disposition for a consumer this branch has not converted. + var stage7Source = BuildStage7PerRunSource( + ctx.Get().Value, ctx.Config, ctx); + if (stage7Source == null) + { + ctx.Publish(new RescoredEntries(_perFileEntries)); + } + else + { + var buffer = _perFileEntries; + ctx.Publish(new RescoredEntries(buffer, + () => MaterializeAllFromSource(buffer, stage7Source, ctx), stage7Source)); + } var bundle = ctx.Get().Value; if (bundle != null) @@ -2011,6 +2030,92 @@ private static Func BuildPerRunHydrate( }; } + /// + /// The per-run source Stage 7 folds through on the --task SecondPassFDR merge: + /// refill ONE run's post-compaction survivors from its own reconciled parquet and + /// 1st-pass sidecar, so the join holds one run at a time instead of all of them + /// (issue #4486). + /// + /// The sibling of , one leg over. That one feeds + /// the rescore loop and returns a whole RunRescoreInputs - actions, gap-fill, + /// refined calibration - because a rescore needs them. A join runs no rescore, so this + /// one returns nothing and refills the caller's list in place; the only state it needs + /// is the pair every refill shares, the analysis-wide retained base_id set and the + /// 1st-pass experiment records, both read ONCE here rather than per run. + /// + /// Null when this leg may not stream, which is the same predicate the + /// --input-scores load consulted when it decided to publish empty per-run lists. + /// Reading it in both places rather than inferring from the lists' emptiness is + /// deliberate: an inference would turn a disagreement between the two sites into a run + /// that folds over 446 empty lists and writes an empty .blib, which is the + /// failure shape this codebase keeps paying for. + /// + /// + /// Fill EVERY run's list through the per-run source - the whole-run build behind + /// RescoredEntries.Value on the streamed second-pass leg. + /// + /// It exists so that a consumer this conversion has not reached still gets a + /// correct pool rather than 446 empty lists, and it is deliberately the same source the + /// fold uses, applied to every run instead of one at a time. Reported, and reported as + /// the expense it is: reaching this line means something asked for the whole pool on the + /// one leg that was arranged not to need it. + /// + private static void MaterializeAllFromSource( + List>> buffer, + Action> source, PipelineContext ctx) + { + ctx.LogWarning(string.Format( + @"Second-pass join: a consumer asked for the whole-run survivor pool, so all " + + @"{0} run(s) are being materialized at once. This is the O(runs x entries) peak " + + @"the per-run fold exists to avoid.", buffer.Count)); + using (var progress = new ProgressReporter(string.Format( + @"Materializing survivors for {0} run(s)", buffer.Count), buffer.Count)) + { + int done = 0; + foreach (var kv in buffer) + { + progress.Report(++done); + source(kv.Key, kv.Value); + } + } + } + + private static Action> BuildStage7PerRunSource( + IReadOnlyDictionary perFileParquetPaths, + OspreyConfig config, + PipelineContext ctx) + { + if (!ScoringTaskShared.CanStreamStage7Join(config)) + return null; + var retainedBaseIds = ScoringTaskShared.ReadRetainedBaseIds(config, out _); + if (retainedBaseIds == null) + return null; + var experimentRecords = FdrExperimentSidecar.ReadMap( + FdrExperimentSidecar.PathFor(config.OutputBlib, + ScoringTaskShared.ArtifactSiblingPath(config), FdrScoresSidecar.Pass.FirstPass), + FdrScoresSidecar.Pass.FirstPass); + // Says which shape Stage 7 took, for the reason its rescore sibling gives: without + // it the only evidence is a memory profile, and "the gate is green so the new path + // must have run" is the inference that lets a resident path pass as a streamed one. + ctx.LogInfo(string.Format( + @"Second-pass join: folding over {0} run(s), each rebuilt from its own artifacts " + + @"and dropped (no all-runs survivor pool; {1} retained base_id(s) read once).", + perFileParquetPaths.Count, retainedBaseIds.Count)); + var sequencePool = ctx.Get().Value; + return (fileName, survivors) => + { + if (!perFileParquetPaths.TryGetValue(fileName, out string parquetPath)) + { + throw new InvalidDataException(string.Format( + @"Second-pass join hydrate: no scores parquet path published for {0}", + fileName)); + } + RescoreHydration.RefillOneRunSurvivors(fileName, parquetPath, survivors, + retainedBaseIds, experimentRecords, + (name, path) => ParquetScoreCache.LoadFdrStubsFromParquet(path, null, sequencePool)); + }; + } + /// /// One run's reconciliation targets, projected through the same /// the join path uses so the two cannot diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs index 660359d2a4..6d07d504fe 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs @@ -1382,10 +1382,19 @@ private FdrProjectionSet LoadJoinOnlyScores( // the same whether the process was handed 1 run or 446, because an HPC node holding // one run cannot pay for the other 445 and must still produce identical output. The // all-runs path below stays for the tasks that genuinely join. - if (ScoringTaskShared.CanHydratePerRun(config)) + // The SAME early return for the reconciled-input merge, whose consumer is Stage 7's + // fold rather than the rescore loop (issue #4486). Both legs want exactly this: the + // run names, their parquet paths and their calibrations, and no rows. What differs + // is only which downstream loop refills a run and drops it - so the predicate is a + // sibling rather than a widening of the one above, and the log line names the + // consumer so a reader of the run log can tell which leg took this path. + bool perRunRescore = ScoringTaskShared.CanHydratePerRun(config); + bool perRunJoin = !perRunRescore && ScoringTaskShared.CanStreamStage7Join(config); + if (perRunRescore || perRunJoin) { LoadJoinOnlyPerRunNames(config, perFileEntries, perFileParquetPaths, - perFileCalibrations, perFileIsolationMz, ctx); + perFileCalibrations, perFileIsolationMz, + perRunJoin ? @"the second-pass join" : @"the rescore", ctx); if (ctx.Diagnostics?.CalibrationOnly ?? false) OspreyDiagnosticsLog.ExitAfterDump(@"OSPREY_CALIBRATION_ONLY"); return null; @@ -1740,11 +1749,12 @@ private static void LoadJoinOnlyPerRunNames( Dictionary perFileParquetPaths, ConcurrentDictionary perFileCalibrations, ConcurrentDictionary> perFileIsolationMz, + string consumer, PipelineContext ctx) { ctx.LogInfo(string.Format( - @"--input-scores: {0} run(s) will be hydrated one at a time by the rescore; " + - @"no all-runs pre-load.", config.InputScores.Count)); + @"--input-scores: {0} run(s) will be hydrated one at a time by {1}; " + + @"no all-runs pre-load.", config.InputScores.Count, consumer)); foreach (string parquetPath in config.InputScores) { string fileName = Path.GetFileNameWithoutExtension( diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs index 666b442fc7..058fd3ba7f 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs @@ -536,9 +536,65 @@ internal sealed class RescoredEntries : PerFileEntries /// private bool _streamed; + /// + /// Applied to each file right after the per-file source has filled it, so a consumer + /// that runs LATER in the stage sees the state the stage has reached rather than the + /// state on disk when the fold started. + /// + /// This is what makes a re-enumerable stream equivalent to a resident pool that + /// is stamped in place. On the resident path a pass-2 overlay or an experiment-q floor + /// is written onto the entries and every later pass reads it; on the streamed path the + /// entries are gone, so the same fact has to be re-applied to each run as it is + /// rebuilt. Both arms end up applying the identical operation to the identical rows - + /// once per run either way - which is why the two produce the same bytes. + /// + /// Added to by the stage, in the order the stage computes the facts: the + /// second-pass sidecar overlay once the sidecars are final, then the experiment-q floors + /// once they have been folded. Order is the point - the floors raise a value the sidecar + /// overlay has just written - so they compose in call order rather than replacing one + /// another. + /// + private Action> _postMaterialize; + + /// + /// True when this milestone has a per-file source, so a consumer may fold through + /// without the buffer holding every file at once - and, the + /// half that callers actually branch on, so a stage knows whether a fact it computes + /// has to be re-applied per file () or can simply be + /// stamped onto a pool that is going to stay. + /// + public bool Streams => _materializeFile != null; + /// The buffer already at its post-rescore state - nothing deferred. public RescoredEntries(List>> value) : base(value) { } + /// + /// Append to the per-file overlay described on , so it + /// runs after everything already installed. + /// + /// REFUSED on a run with no per-file source, rather than ignored. There the + /// entries are the pool and the caller must stamp them in place as it always did; an + /// overlay installed and never invoked would leave a stage believing it had applied + /// something it had not, which is the class of defect this whole area keeps producing. + /// Callers branch on and do one or the other. + /// + public void AddPostMaterialize(Action> overlay) + { + if (overlay == null) + throw new ArgumentNullException(nameof(overlay)); + if (_materializeFile == null) + { + throw new InvalidOperationException( + @"RescoredEntries.AddPostMaterialize was called on a milestone with no per-file " + + @"source, where nothing would ever invoke it. Apply the operation to the " + + @"resident buffer instead; branch on Streams."); + } + var existing = _postMaterialize; + _postMaterialize = existing == null + ? overlay + : (name, entries) => { existing(name, entries); overlay(name, entries); }; + } + /// /// The run's files, one at a time, for a consumer that ITERATES and does not retain. /// @@ -631,6 +687,7 @@ public IEnumerable>> StreamFiles() foreach (var kv in base.Value) { _materializeFile(kv.Key, kv.Value); + _postMaterialize?.Invoke(kv.Key, kv.Value); yield return kv; // Dropped as soon as the consumer's foreach body returns. TrimExcess too: // Clear leaves the backing array at its high-water capacity, which for a CHS @@ -640,6 +697,60 @@ public IEnumerable>> StreamFiles() kv.Value.TrimExcess(); } } + + /// + /// ONE named file's post-rescore survivors, for a consumer that is driven by something + /// other than this buffer's order - the streamed pass-2 competition asks for the file + /// the FDR layer has just decided to read next, not for "the next one". + /// + /// The list is the buffer's own, so a caller that stamps entries stamps what the + /// resident path would have. On the streamed source it has just been filled from disk + /// and the caller owns dropping it (); on the resident source it + /// is already filled and must NOT be dropped, which is why the two halves are separate + /// calls rather than one scoped helper - only the caller knows whether it is done with + /// the file or merely finished one of several passes over it. + /// + /// An unknown name returns an empty list rather than throwing: that is what the + /// resident lookup this replaces did for a file the buffer never held, and turning it + /// into a throw here would convert a tolerated input-naming drift (reported in its own + /// words upstream) into an abort hours into a run. + /// + public List MaterializeFile(string fileName) + { + foreach (var kv in base.Value) + { + if (!string.Equals(kv.Key, fileName, StringComparison.Ordinal)) + continue; + if (_materializeFile != null) + { + _materializeFile(kv.Key, kv.Value); + _postMaterialize?.Invoke(kv.Key, kv.Value); + } + return kv.Value; + } + return new List(); + } + + /// + /// Release one file materialized by . A no-op on the + /// resident source, where the buffer IS the pool and dropping it would destroy the only + /// copy - the same asymmetry encodes by falling back to + /// . + /// + public void DropFile(string fileName) + { + if (_materializeFile == null) + return; + foreach (var kv in base.Value) + { + if (!string.Equals(kv.Key, fileName, StringComparison.Ordinal)) + continue; + _streamed = true; + kv.Value.Clear(); + kv.Value.TrimExcess(); + return; + } + } } /// diff --git a/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs b/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs index b28adb89b4..c03d48aabc 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs @@ -811,6 +811,63 @@ public static RunRescoreInputs HydrateOneRun( }; } + /// + /// ONE run's post-compaction survivors, refilled IN PLACE, for a join that folds over the + /// runs one at a time and drops each - the Stage 7 shape (issue #4486). + /// + /// The same three steps takes - load the run's stubs, + /// overlay its 1st-pass sidecar, compact to the analysis-wide retained set - and NOT the + /// fourth. Stage 7 runs no rescore, so it needs neither the planned actions nor the + /// reconciliation.json they come from, and reading that envelope per run would put + /// ~6 MB of join-wide first_pass_base_ids through this call 446 times to answer a + /// question nobody asks. Sharing the whole of HydrateOneRun would have been the + /// tidier-looking choice and the more expensive one. + /// + /// Refills the caller's list rather than returning a new one. That list is + /// the shared backing store every PerFileEntries milestone wraps, so replacing the + /// reference would leave the published milestones pointing at the old one - the same rule + /// the Stage 6 refill follows. Contents are transient; identity is not. + /// + /// Equivalence with the resident path is what makes a streamed Stage 7 produce the + /// same bytes: this is the state HydrateCompactedStreaming leaves a run in, reached + /// by the same calls in the same order. The difference is only how long the list lives. + /// + public static void RefillOneRunSurvivors( + string fileName, + string parquetPath, + List survivors, + HashSet retainedBaseIds, + IReadOnlyDictionary experimentRecords, + Func> loadStubs) + { + if (survivors == null) + throw new ArgumentNullException(nameof(survivors)); + if (retainedBaseIds == null) + throw new ArgumentNullException(nameof(retainedBaseIds)); + if (loadStubs == null) + throw new ArgumentNullException(nameof(loadStubs)); + + var stubs = loadStubs(fileName, parquetPath); + if (stubs == null) + { + throw new InvalidDataException(string.Format( + "RefillOneRunSurvivors: no stubs loaded for {0}", fileName)); + } + string syntheticInput = SyntheticInputFromParquet(parquetPath); + // Overlay then compact, in that order, for the reason the two siblings state: the + // sidecar covers the whole PRE-compaction row set, so the filter has to name the + // records that legitimately have no entry to land on and leave every other miss + // reportable as the parquet drift it is. + OverlayFirstPassSidecar(syntheticInput, fileName, stubs, + nameof(RefillOneRunSurvivors), experimentRecords, + id => !retainedBaseIds.Contains(id & ScoringTaskShared.BASE_ID_MASK)); + stubs.RemoveAll(e => !retainedBaseIds.Contains(e.EntryId & ScoringTaskShared.BASE_ID_MASK)); + + survivors.Clear(); + survivors.AddRange(stubs); + survivors.TrimExcess(); + } + /// /// Overlay the first-pass FDR statistics onto : the RUN-scope /// SVM score, run q-values and PEP from <stem>.1st-pass.fdr_scores.bin, diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index 389da47b69..b007e20a63 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -1,4 +1,4 @@ -/* +/* * Original author: Brendan MacLean , * MacCoss Lab, Department of Genome Sciences, UW * AI assistance: Claude Code (Claude Opus 4.8) @@ -435,6 +435,48 @@ internal static bool CanHydratePerRun(OspreyConfig config) return !string.IsNullOrEmpty(path) && RetainedBaseIdSidecar.IsCurrentFormat(path); } + /// + /// True when the --task SecondPassFDR merge may hand Stage 7 a per-run source + /// instead of every run's survivors at once. + /// + /// This is 's answer for the OTHER leg, and the two + /// are deliberately separate predicates rather than one with a wider admission. They + /// name different consumers: that one asks whether the RESCORE can hydrate a run at a + /// time, and excludes ExpectReconciledInput because a Stage 7 node runs no + /// rescore; this one asks whether the JOIN can fold a run at a time, and admits only + /// that leg. Widening the first would have told the rescore it may stream on a leg where + /// it does not run at all. + /// + /// Three requirements, and the third is the one that is easy to miss. The leg has + /// to be the reconciled-input merge, whose parquets already hold the survivor subset. + /// No consumer may read PIN features off these stubs + /// (PerFileScoringTask.NeedsResidentPool: --fdrbench-pass 1, a + /// non-Percolator FDR method, OSPREY_FDR_PROJECTION=0) - a streamed pool drops + /// the entries those consumers index. And the analysis-wide retained base_id summary has + /// to be on disk, because it IS the compaction predicate every refill applies; without + /// it a refilled run would carry the pre-compaction pool and the fold would run over a + /// set ~52x too large. Its absence returns false here rather than failing, for the + /// reason its sibling gives. + /// + internal static bool CanStreamStage7Join(OspreyConfig config) + { + if (!config.ExpectReconciledInput || !OspreyEnvironment.Stage7Stream) + return false; + if (PerFileScoringTask.NeedsResidentPool(config, OspreyEnvironment.UseFdrProjection)) + return false; + // --model-diagnostics is the fourth requirement, and it is a CURRENT limitation + // rather than a property of the leg. The pass-2 report builders index their files by + // position and revisit a file across two loops, so they need a list and not a + // stream; the fold that removes the need is the accumulator the pass-1 report + // already uses. Declining here rather than letting the report leg stream and then + // silently pull the whole pool back through .Value, which is the same peak reached + // by a longer route and with nothing in the log to say so. + if (config.ModelDiagnostics) + return false; + string path = RetainedBaseIdSidecar.PathFor(config.OutputBlib, ArtifactSiblingPath(config)); + return !string.IsNullOrEmpty(path) && RetainedBaseIdSidecar.IsCurrentFormat(path); + } + /// /// Read the analysis-wide retained base_id summary FirstPassFDR left behind, or return /// null with set to an operator-facing message naming the diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 74ee82d641..e046fe4c5f 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -1,4 +1,4 @@ -/* +/* * Original author: Brendan MacLean , * MacCoss Lab, Department of Genome Sciences, UW * AI assistance: Claude Code (Claude Opus 5) @@ -241,6 +241,11 @@ public override string ValidityKey(PipelineContext ctx) + OspreyEnvironment.ExperimentAggValidityKeySuffix() + OspreyEnvironment.Pass2QValueValidityKeySuffix() + OspreyEnvironment.TrainSampleValidityKeySuffix() + // And the fold-versus-resident arm, on the argument its Stage 6 twin makes: the + // two are supposed to write byte-identical .blib and 2nd-pass sidecars, and an + // in-place A/B that adopted the other arm's outputs would report that identity + // without ever testing it. + + OspreyEnvironment.Stage7StreamValidityKeySuffix() + LibraryFragmentRelease.ValidityKeySuffix(ctx); } @@ -327,9 +332,14 @@ public override bool Run(PipelineContext ctx) stale.Count, rescored.FileCount, string.Join(", ", stale))); } - var perFileEntries = rescored.Value; + // NO .Value here any more (#4486). Every consumer below folds through + // RescoredEntries.StreamFiles, which on the per-run source rebuilds one run, hands + // it over and drops it - so the probe that used to measure "the survivor pool built" + // now measures a stage that never builds one. It is kept, and kept in place, because + // the whole #4486 series is quoted against it: on the streamed arm it should read + // flat against stage7-inherited, and a jump here is the pool coming back. ProfilerHooks.LogManagedHeapAfterGcIfEnabled(ctx.LogInfo, @"stage7-pool", - string.Format(@"(post-GC, survivor pool built, files={0})", perFileEntries.Count)); + string.Format(@"(post-GC, entering the fold, files={0})", rescored.FileCount)); // Beside the probe that measures the pool, because it explains part of it: a // distinct count still equal to the seed means the survivors' sequences are the // library's own instances rather than one string per observation (#4486). @@ -361,6 +371,12 @@ public override bool Run(PipelineContext ctx) pass2Contributions = Pass2FdrSidecar.ComputeAndPersist( ctx, AnyReconciledParquet(config), rescored, perFileParquetPaths, Name, ValidityKey(ctx)); + // From here on, every fold must see the SECOND pass's answer. On the resident pool + // ComputeAndPersist has just stamped it onto the entries; on the streamed one the + // entries it stamped are gone, so the same overlay is installed as a per-run hook + // and re-applied to each run as the folds below rebuild it. No-op on the resident + // arm, which is what keeps the two arms one code path rather than two. + Pass2FdrSidecar.InstallStreamedPass2Overlay(ctx, rescored, Name, ValidityKey(ctx)); // The substep the 2026-07-31 characterization on #4486 located the churn in: // it reloads every file's reconciled features, so the pre-GC line carries the // transient reload peak and the post-GC line what survives it. @@ -401,7 +417,7 @@ public override bool Run(PipelineContext ctx) // experiment q with no surviving run support -- reported with no run-level ID (the // blib ID-line artifact). Re-clamping here, against the run q's actually written to // the blib, restores "reported => some run genuinely passed" for the final output. - PercolatorEngine.ClampExperimentQToBestRun(perFileEntries); + ReclampExperimentQToBestRun(rescored); // Write output blib - unless this is a diagnostics-only regeneration, whose whole // contract is that it touches no artifact but the report. @@ -446,7 +462,7 @@ public override bool Run(PipelineContext ctx) var swFdrBench = Stopwatch.StartNew(); var pairing = EntrapmentPairing.Build(libraryById, config.DecoyPairingManifestPath); var benchResult = FdrBenchInputWriter.WritePeptideInput( - benchPath, perFileEntries, libraryById, config.FdrLevel, + benchPath, rescored.StreamFiles(), libraryById, config.FdrLevel, config.FdrBenchPerRun, pairing.ExcludedEntrapment); // Emit the corrected pairing manifest from the same library so FDRBench // classifies every reported peptide and drops nothing (feed FDRBench -pep with this). @@ -485,14 +501,63 @@ public override bool Run(PipelineContext ctx) if (OspreyEnvironment.Pass2ProteinCompact && ctx.TryGet(out var pcStratum)) stratumBaseIds = pcStratum.BaseIds; + // The ONE Stage 7 consumer still handed the whole pool, and the only one that + // cannot simply be re-pointed at the stream: the pass-2 data builders index + // their files BY POSITION and revisit a file across two loops + // (ModelDiagnosticsData.CoAssignment), so an enumerable that rebuilds a run per + // pass would rebuild the same run several times per view. The fold it wants is + // the one FirstPassFDR's pass-1 accumulator already is, and giving pass 2 the + // same accumulator is the fix - not a wider parameter type. Until then + // CanStreamStage7Join declines the report leg outright, so reaching this line + // means the pool is resident and .Value is what it has always been. ModelDiagnosticsReport.WritePass2AndFinalize( - perFileEntries, pass2Contributions, libraryById, config, ctx.LogInfo, + rescored.Value, pass2Contributions, libraryById, config, ctx.LogInfo, stratumBaseIds, ValidityKey(ctx)); } return true; } + /// + /// Re-clamp experiment q to each precursor's best run q, as a FOLD then an APPLY. + /// + /// The floors are whole-experiment and the application is per row, which is why + /// exposes the two halves separately. Folding them over a + /// stream visits every run and holds two O(distinct) maps - 45,724 precursors at 257 CHS + /// runs, not 137 M entries - so this genuinely whole-experiment step is one of the folds + /// the architecture admits in a join, not a reason to hold the pool. + /// + /// The apply half differs by arm and cannot not. On the resident pool the floors + /// are stamped onto the entries once and every later gate reads them. On the streamed + /// one there are no entries between passes, so the apply is installed as a per-run + /// overlay and re-applied to each run as the blib gates rebuild it - after the pass-2 + /// sidecar overlay, because a floor raises the value that overlay has just written. + /// Composition order is the correctness argument, and it is why these go in as two + /// AddPostMaterialize calls in this order rather than one. + /// + private static void ReclampExperimentQToBestRun(RescoredEntries rescored) + { + var minRunBothByEntryId = new Dictionary(); + var minRunBothByPeptide = new Dictionary<(string ModifiedSequence, bool IsDecoy), double>(); + foreach (var kvp in rescored.StreamFiles()) + { + PercolatorEngine.AccumulateExperimentQFloors( + kvp.Value, minRunBothByEntryId, minRunBothByPeptide); + } + if (rescored.Streams) + { + rescored.AddPostMaterialize((fileName, entries) => + PercolatorEngine.ApplyExperimentQFloors( + entries, minRunBothByEntryId, minRunBothByPeptide)); + return; + } + foreach (var kvp in rescored.Files()) + { + PercolatorEngine.ApplyExperimentQFloors( + kvp.Value, minRunBothByEntryId, minRunBothByPeptide); + } + } + /// /// Drop Fragments from every library entry outside the final per-file pool, /// keeping the identity fields on all of them. See @@ -523,7 +588,7 @@ private void ReleaseUnscorableLibraryFragments( // the files one at a time and drop each. While something else still reads the // whole-run buffer, Files() yields from it and this costs nothing; once nothing // does, it is one file resident at a time (#4486). - var retained = LibraryFragmentRelease.BuildRetainedBaseIds(rescored.Files()); + var retained = LibraryFragmentRelease.BuildRetainedBaseIds(rescored.StreamFiles()); int released = LibraryFragmentRelease.ReleaseFragments(fullLibrary, retained); ctx.LogInfo(string.Format( @"Released library fragments for {0} of {1} entries ({2} base_ids retained for the reported pool)", @@ -556,7 +621,7 @@ private void RunProteinFdr( PipelineContext ctx) { var result = ProteinFdrEngine.RunSecondPass( - rescored.Files(), fullLibrary, config, ctx.LogInfo); + rescored.StreamFiles(), fullLibrary, config, ctx.LogInfo); // The 2nd-pass sidecar was written BEFORE this protein FDR ran - it is one of its // inputs - so the protein column it carries is still the pass-1 value at this point. @@ -723,10 +788,10 @@ private void WriteBlibOutput( // charge state (lowest experiment_precursor_qvalue) as a // representative. // Streamed: both gates fold to O(distinct) and retain nothing. - var passingPeptides = ComputePassingPeptides(rescored.Files(), config, nFiles); + var passingPeptides = ComputePassingPeptides(rescored.StreamFiles(), config, nFiles); var passingPrecursors = ComputePassingPrecursors( - rescored.Files(), config, passingPeptides, nFiles, out int nFallback); + rescored.StreamFiles(), config, passingPeptides, nFiles, out int nFallback); if (nFallback > 0) { ctx.LogInfo(string.Format( @@ -739,7 +804,7 @@ private void WriteBlibOutput( // everything after this line works on ~14 M values instead of holding 137 M // entries alive to read eight fields off them (#4486). var passingEntries = CollectPassingEntries( - rescored.Files(), passingPrecursors, nFiles, ctx.Get().Value, + rescored.StreamFiles(), passingPrecursors, nFiles, ctx.Get().Value, out var bestByPrecursor); ctx.LogInfo(string.Format( From 5ed90f3e2213350df367d886ec32ccd13fd082a8 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Sun, 6 Sep 2026 23:50:33 -0700 Subject: [PATCH 08/30] Finished the Stage 7 fold: the last pool reader, and the silence it left * OspreyReportWriter.WriteSummary still walked the pool, so a 446-run fold ran for 40 minutes and then threw at the per-replicate summary. The guard did its job - "RescoredEntries.Value was read after StreamFiles dropped the survivor pool", naming the consumer - rather than writing a report over 446 empty runs. Its loop consumes one run and retains nothing, so it folds unchanged * Straight-through Stage 7 keeps its resident pool, and this is a property of the materializer rather than a preference. MaterializeRescoredFile is ONE-SHOT: it overlays the reconciled parquet and appends gap-fill rows, so a re-enumerating fold duplicates them - the same "run-once, and a failed build stays failed" rule RescoredEntries states for the whole-run build. Handing it over produced a straight-through Stellar run that exited 1. The leg with a repeatable source is the reconciled-input merge, and only it streams * The pass-2 competition reported progress from ReadFile, which runs ONLY where a file has to be recomputed - so on the path this work exists to produce, every file answered by the worker, the counter never moved and the phase was one silent block: 654 s at 446 runs, starting immediately after the line announcing the fold. Moved to BeginFile, which is the call that always happens once per file * StreamFiles takes a label and reports per run. On a streamed source every pass REBUILDS each run from disk, so folds that used to walk memory in seconds now run for minutes; unreported, they are silences in the middle of a multi-hour stage Measured at 446 runs, same bed and recipe as the "before", through protein FDR: before peak 68.0 GB managed / 70.5 GB private floor RISING +71 MB/file killed at 381/446 after peak 18.5 GB managed / 33.5 GB private floor FALLING -4 MB/file runs to completion The floor is the result, not the peak: rising was O(files) accumulation, and it is gone. 605 tests (604 pass, 1 pre-existing skip), inspection zero-warning. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- pwiz_tools/Osprey/Osprey-workflow.html | 1 + .../Osprey/Osprey.Tasks/OspreyReportWriter.cs | 2 +- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 10 ++- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 70 +++++++++++++------ .../Osprey/Osprey.Tasks/PerFileScoringTask.cs | 12 ++++ .../Osprey/Osprey.Tasks/PipelineByproducts.cs | 44 ++++++++---- .../Osprey/Osprey.Tasks/RescoreHydration.cs | 36 +++++++--- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 8 +-- .../Osprey/docs/00-pipeline-architecture.md | 28 +++++++- 9 files changed, 159 insertions(+), 52 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey-workflow.html b/pwiz_tools/Osprey/Osprey-workflow.html index 55f59c22af..750abd1b3a 100644 --- a/pwiz_tools/Osprey/Osprey-workflow.html +++ b/pwiz_tools/Osprey/Osprey-workflow.html @@ -509,6 +509,7 @@

Osprey DIA pipeline workflow

in  <stem>.scores-reconciled.parquet (via --input-scores; falls back to <stem>.scores.parquet), .2nd-pass.fdr_scores.bin (worker) in  <blib-stem>.1st-pass.fdr_experiment.bin, <stem>.1st-pass.model.json, .1st-pass.stratum.json, <stem>.reconciliation.json, .calibration.json out <output>.blib, <blib-stem>.2nd-pass.fdr_experiment.bin, <stem>.2nd-pass.fdr_scores.bin (where no worker ran) · validity <out>.SecondPassFDR.osprey.task + holds O(distinct) · folds the runs one at a time, rebuilding each from its own artifacts and dropping it relay: every run's reconciled set, with its .osprey.task stamps diff --git a/pwiz_tools/Osprey/Osprey.Tasks/OspreyReportWriter.cs b/pwiz_tools/Osprey/Osprey.Tasks/OspreyReportWriter.cs index 5accffc468..f30eff3eb0 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/OspreyReportWriter.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/OspreyReportWriter.cs @@ -263,7 +263,7 @@ private static void WriteSummary( string.Format(@"Per-replicate protein FDR over {0} run(s)", nFiles), nFiles, string.Empty, ProgressReporter.IO_INTERVAL_SECONDS)) { - foreach (var kvp in rescored.Files()) + foreach (var kvp in rescored.StreamFiles()) { progress.Report(++runIdx); AccumulatePrecursorsPeptides(kvp.Value, level, config, runLevel: false, diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index 270e92efc1..b886763cb2 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -1947,6 +1947,14 @@ List LoadOneFile(string fileKey) // reads the file's 1st-pass population. void BeginFile(string fileKey) { + // Reported HERE, not in ReadFile. ReadFile runs only where this pass has to + // RECOMPUTE a file's competition, so on the path the move exists to produce - + // every file answered by the worker - it never ran, the counter never moved, + // and the phase was a single silent block: 654 s at 446 runs, immediately + // after the line announcing that the fold was reading the worker's answers. + // BeginFile is the call that always happens, once per file, which is what a + // per-file progress signal has to be attached to. + progress.Report(++nRead); currentKey = fileKey; currentEntries = LoadOneFile(fileKey); currentWorkerRecords = null; @@ -1976,8 +1984,6 @@ void BeginFile(string fileKey) survivorIds, pass1Records, out uint[] eids, out double[] scs, out var fileScores); nScored += fileScores.Count; - - progress.Report(++nRead); return (eids, scs, fileScores, survivorIds); } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index a8c0f4e91f..e90e0b61cd 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -318,26 +318,24 @@ public override bool Run(PipelineContext ctx) // it dropped, so streaming there would destroy the only copy of the survivors on // its way past. Null leaves StreamFiles walking the resident buffer, which is what // the OSPREY_STAGE6_STREAM_SURVIVORS=0 A/B oracle needs it to do. - Action> materializeOneFile = null; - if (survivorLoader != null) - materializeOneFile = MaterializeOneFile; - var rescored = new RescoredEntries(_perFileEntries, () => BuildRescoredPool(ctx), - materializeOneFile); + // NO per-file source on this path, and the reason is a property of the materializer + // rather than a preference. MaterializeRescoredFile is ONE-SHOT: it overlays the + // reconciled parquet and appends gap-fill rows, so calling it twice for one run + // duplicates them - which is the same "run-once, and a failed build stays failed" + // rule RescoredEntries' own remarks state for the whole-run build. A Stage 7 fold + // re-enumerates, so it needs a source that can rebuild a run from disk repeatedly + // and identically; this one cannot, and handing it over produced a straight-through + // Stellar run that exited 1 on AssertSidecarDescribesPool. + // + // The leg that has such a source is the reconciled-input merge, where + // BuildStage7PerRunSource supplies it (Rehydrate, below). Straight-through Stage 7 + // therefore keeps its resident pool for now. That is the honest state and not a + // hidden one: making this materializer idempotent - or having it rebuild from the + // reconciled parquet alone, which already holds the merged gap-fill rows - is what + // extends the fold to this leg, and it is separate work. + var rescored = new RescoredEntries(_perFileEntries, () => BuildRescoredPool(ctx)); ctx.Publish(rescored); - void MaterializeOneFile(string fileName, List entries) - { - // Same refusal as BuildRescoredPool, and for the same reason: every guess - // available before Run has decided produces a wrong reported set rather than - // an error. - if (_poolPlan == null) - { - throw new InvalidOperationException( - @"RescoredEntries was streamed before PerFileRescoring decided how to build the survivor pool."); - } - MaterializeRescoredFile(ctx, _poolPlan, fileName, entries); - } - // Self-gate: rescore + reconciliation only run when there is // planning state to act on AND the rescore hasn't already been // done upstream. State comes from either FirstPassFdrTask's @@ -2094,13 +2092,42 @@ private static Action> BuildStage7PerRunSource( FdrExperimentSidecar.PathFor(config.OutputBlib, ScoringTaskShared.ArtifactSiblingPath(config), FdrScoresSidecar.Pass.FirstPass), FdrScoresSidecar.Pass.FirstPass); + // Which runs already carry a second-pass answer, decided ONCE from the artifacts + // themselves (a header probe per run, no records read). A run in this set is + // rebuilt WITHOUT opening any first-pass file, which is the Boundary 3 -> 4 + // contract: its .2nd-pass.fdr_scores.bin holds every scalar the join reads, and the + // experiment-scope columns come from the analysis-wide sidecar. Probed here rather + // than per refill because each run is rebuilt once per fold pass, and the answer + // cannot change while the stage runs. + var inputByName = new Dictionary(StringComparer.Ordinal); + if (config.InputFiles != null) + { + foreach (string inputFile in config.InputFiles) + inputByName[Path.GetFileNameWithoutExtension(inputFile)] = inputFile; + } + var haveSecondPass = new HashSet(StringComparer.Ordinal); + foreach (var kv in perFileParquetPaths) + { + if (!inputByName.TryGetValue(kv.Key, out string inputFile)) + continue; + if (FdrScoresSidecar.IsCurrentFormat( + FdrScoresSidecar.Pass2Path(inputFile), FdrScoresSidecar.Pass.SecondPass)) + { + haveSecondPass.Add(kv.Key); + } + } // Says which shape Stage 7 took, for the reason its rescore sibling gives: without // it the only evidence is a memory profile, and "the gate is green so the new path // must have run" is the inference that lets a resident path pass as a streamed one. + // The second count is the boundary claim, said out loud: an orchestrator that + // trimmed the first-pass sidecars is entitled to know how many runs would have + // needed them. ctx.LogInfo(string.Format( @"Second-pass join: folding over {0} run(s), each rebuilt from its own artifacts " + - @"and dropped (no all-runs survivor pool; {1} retained base_id(s) read once).", - perFileParquetPaths.Count, retainedBaseIds.Count)); + @"and dropped (no all-runs survivor pool; {1} retained base_id(s) read once). " + + @"{2} of {0} run(s) carry a current 2nd-pass sidecar and are rebuilt without " + + @"opening any 1st-pass file.", + perFileParquetPaths.Count, retainedBaseIds.Count, haveSecondPass.Count)); var sequencePool = ctx.Get().Value; return (fileName, survivors) => { @@ -2112,7 +2139,8 @@ private static Action> BuildStage7PerRunSource( } RescoreHydration.RefillOneRunSurvivors(fileName, parquetPath, survivors, retainedBaseIds, experimentRecords, - (name, path) => ParquetScoreCache.LoadFdrStubsFromParquet(path, null, sequencePool)); + (name, path) => ParquetScoreCache.LoadFdrStubsFromParquet(path, null, sequencePool), + !haveSecondPass.Contains(fileName)); }; } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs index 6d07d504fe..6dd4808bf4 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs @@ -1886,6 +1886,18 @@ private bool HydrateRescoreBundleIfPresent( bool hasReconSidecars, PipelineContext ctx) { + // NOTHING to hydrate when the runs are hydrated one at a time - by the rescore + // loop or, on the reconciled-input merge, by Stage 7's fold. The loader published + // one EMPTY list per run and read no rows, so the batch overlay below would read + // every run's 1st-pass sidecar and fail to place a single record: "failed to + // overlay .1st-pass.fdr_scores.bin", naming a file that is present and correct. + // The bundle it builds is the all-runs structure both per-run shapes exist not to + // build, and both leave _rescoreInputs null so their consumers hydrate per run. + if (ScoringTaskShared.CanHydratePerRun(config) || + ScoringTaskShared.CanStreamStage7Join(config)) + { + return true; + } if (hasReconSidecars) { // Already hydrated when the loader took the file-count-bounded streaming diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs index 058fd3ba7f..015b92704f 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs @@ -673,7 +673,13 @@ public override List>> Value /// Falls back to the resident walk when this run has no per-file source, so the /// oracle paths are unaffected. /// - public IEnumerable>> StreamFiles() + /// What this fold is doing, for the per-run progress line. Supply + /// it: on a streamed source each pass REBUILDS every run from disk, so a fold that used + /// to walk memory in seconds now runs for minutes, and an unreported one is a silence + /// in the middle of a multi-hour stage - the shape this codebase has repeatedly had to + /// go back and fix. Null suppresses the line, which is right only for a fold that + /// already reports its own progress. + public IEnumerable>> StreamFiles(string label = null) { if (_materializeFile == null) { @@ -681,20 +687,30 @@ public IEnumerable>> StreamFiles() yield return kv; yield break; } - // base.Value, not Value: the pairs and their (empty) lists are what we materialize - // INTO, so reaching them must not trigger the whole-run build this method exists - // to replace - nor trip the _streamed guard on a second pass. - foreach (var kv in base.Value) + // Disposed by the enumerator's own finally, so an abandoned fold closes its + // reporter rather than leaving the heading as the last line in the log. + using (var progress = label == null + ? null + : new ProgressReporter(string.Format(@"{0} over {1} run(s)", label, FileCount), + FileCount, string.Empty, ProgressReporter.IO_INTERVAL_SECONDS)) { - _materializeFile(kv.Key, kv.Value); - _postMaterialize?.Invoke(kv.Key, kv.Value); - yield return kv; - // Dropped as soon as the consumer's foreach body returns. TrimExcess too: - // Clear leaves the backing array at its high-water capacity, which for a CHS - // file is ~648 K references still committed per file. - _streamed = true; - kv.Value.Clear(); - kv.Value.TrimExcess(); + int done = 0; + // base.Value, not Value: the pairs and their (empty) lists are what we + // materialize INTO, so reaching them must not trigger the whole-run build this + // method exists to replace - nor trip the _streamed guard on a second pass. + foreach (var kv in base.Value) + { + progress?.Report(++done); + _materializeFile(kv.Key, kv.Value); + _postMaterialize?.Invoke(kv.Key, kv.Value); + yield return kv; + // Dropped as soon as the consumer's foreach body returns. TrimExcess too: + // Clear leaves the backing array at its high-water capacity, which for a CHS + // file is ~648 K references still committed per file. + _streamed = true; + kv.Value.Clear(); + kv.Value.TrimExcess(); + } } } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs b/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs index c03d48aabc..adaa5e8687 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs @@ -831,6 +831,15 @@ public static RunRescoreInputs HydrateOneRun( /// Equivalence with the resident path is what makes a streamed Stage 7 produce the /// same bytes: this is the state HydrateCompactedStreaming leaves a run in, reached /// by the same calls in the same order. The difference is only how long the list lives. + /// + /// is the Boundary 3 -> 4 contract made a + /// parameter. FALSE for a run that already carries a current + /// .2nd-pass.fdr_scores.bin: that file holds every scalar the join reads, so + /// opening the first-pass one would reach back across a boundary issue #4486 exists to + /// establish - and would do it for every run in the cohort, on precisely the leg where an + /// orchestrator is entitled not to have shipped them. TRUE where there is no second-pass + /// answer yet, which is a mode whose per-run half still runs in the join and is the + /// exception the boundary already documents rather than a new one. /// public static void RefillOneRunSurvivors( string fileName, @@ -838,7 +847,8 @@ public static void RefillOneRunSurvivors( List survivors, HashSet retainedBaseIds, IReadOnlyDictionary experimentRecords, - Func> loadStubs) + Func> loadStubs, + bool overlayFirstPass) { if (survivors == null) throw new ArgumentNullException(nameof(survivors)); @@ -853,14 +863,22 @@ public static void RefillOneRunSurvivors( throw new InvalidDataException(string.Format( "RefillOneRunSurvivors: no stubs loaded for {0}", fileName)); } - string syntheticInput = SyntheticInputFromParquet(parquetPath); - // Overlay then compact, in that order, for the reason the two siblings state: the - // sidecar covers the whole PRE-compaction row set, so the filter has to name the - // records that legitimately have no entry to land on and leave every other miss - // reportable as the parquet drift it is. - OverlayFirstPassSidecar(syntheticInput, fileName, stubs, - nameof(RefillOneRunSurvivors), experimentRecords, - id => !retainedBaseIds.Contains(id & ScoringTaskShared.BASE_ID_MASK)); + if (overlayFirstPass) + { + string syntheticInput = SyntheticInputFromParquet(parquetPath); + // Overlay then compact, in that order, for the reason the two siblings state: + // the sidecar covers the whole PRE-compaction row set, so the filter has to + // name the records that legitimately have no entry to land on and leave every + // other miss reportable as the parquet drift it is. + OverlayFirstPassSidecar(syntheticInput, fileName, stubs, + nameof(RefillOneRunSurvivors), experimentRecords, + id => !retainedBaseIds.Contains(id & ScoringTaskShared.BASE_ID_MASK)); + } + // Kept even where the reconciled parquet is already the survivor subset and this + // removes nothing. It is the analysis-wide compaction predicate, it is the same + // one both siblings apply, and "the parquet is already subset so the filter is + // redundant" is a property of an artifact generation rather than of the format - + // exactly the kind of assumption that turns into a silently larger pool. stubs.RemoveAll(e => !retainedBaseIds.Contains(e.EntryId & ScoringTaskShared.BASE_ID_MASK)); survivors.Clear(); diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index e046fe4c5f..c241be1ecc 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -462,7 +462,7 @@ public override bool Run(PipelineContext ctx) var swFdrBench = Stopwatch.StartNew(); var pairing = EntrapmentPairing.Build(libraryById, config.DecoyPairingManifestPath); var benchResult = FdrBenchInputWriter.WritePeptideInput( - benchPath, rescored.StreamFiles(), libraryById, config.FdrLevel, + benchPath, rescored.StreamFiles(@"Writing FDRBench input"), libraryById, config.FdrLevel, config.FdrBenchPerRun, pairing.ExcludedEntrapment); // Emit the corrected pairing manifest from the same library so FDRBench // classifies every reported peptide and drops nothing (feed FDRBench -pep with this). @@ -539,7 +539,7 @@ private static void ReclampExperimentQToBestRun(RescoredEntries rescored) { var minRunBothByEntryId = new Dictionary(); var minRunBothByPeptide = new Dictionary<(string ModifiedSequence, bool IsDecoy), double>(); - foreach (var kvp in rescored.StreamFiles()) + foreach (var kvp in rescored.StreamFiles(@"Folding experiment-q floors")) { PercolatorEngine.AccumulateExperimentQFloors( kvp.Value, minRunBothByEntryId, minRunBothByPeptide); @@ -588,7 +588,7 @@ private void ReleaseUnscorableLibraryFragments( // the files one at a time and drop each. While something else still reads the // whole-run buffer, Files() yields from it and this costs nothing; once nothing // does, it is one file resident at a time (#4486). - var retained = LibraryFragmentRelease.BuildRetainedBaseIds(rescored.StreamFiles()); + var retained = LibraryFragmentRelease.BuildRetainedBaseIds(rescored.StreamFiles(@"Collecting the reported base_ids")); int released = LibraryFragmentRelease.ReleaseFragments(fullLibrary, retained); ctx.LogInfo(string.Format( @"Released library fragments for {0} of {1} entries ({2} base_ids retained for the reported pool)", @@ -621,7 +621,7 @@ private void RunProteinFdr( PipelineContext ctx) { var result = ProteinFdrEngine.RunSecondPass( - rescored.StreamFiles(), fullLibrary, config, ctx.LogInfo); + rescored.StreamFiles(@"Collecting best scores for protein FDR"), fullLibrary, config, ctx.LogInfo); // The 2nd-pass sidecar was written BEFORE this protein FDR ran - it is one of its // inputs - so the protein column it carries is still the pass-1 value at this point. diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index 6da1410fb1..b269902cb3 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -145,6 +145,25 @@ so where it matters - "both maps are O(distinct) ... nothing here needs a whole- and `StreamingFdr.StreamingFirstPassQ` is the worked example, pinned against its resident twin by a test. +`SecondPassFDR` is the largest one, and it is worth reading as the cautionary case as well +as the worked one, because for a long time every step in it *was* a fold and the stage still +held the pool. The fragment release, the pass-2 competition, protein parsimony, the +experiment-q re-clamp and all three `.blib` gates each reduce to `O(distinct)` and each +visits every run - but the stage was **handed** every run's survivors before the first of +them started, by the `--input-scores` merge, so nothing they did could bring the peak down. +At 446 CHS runs that load reached 68.0 GB and was killed at run 381 with 0.34 GB free, +having computed nothing. **A fold does not bound anything unless its SOURCE is per-run +too**: the runs are now rebuilt one at a time from their own +`.scores-reconciled.parquet` and 1st-pass sidecar, folded, and dropped. + +Two consequences of that shape are worth stating, because they are what a reader will +otherwise trip over. A stage that revisits its runs pays the rebuild once per pass, so +wall-clock trades against memory here exactly as the "memory, not wall clock" target says it +should. And a fact one pass computes and a later pass reads - the second-pass sidecar +overlay, the experiment-q floor - cannot be left stamped on entries that no longer exist, so +it is re-applied to each run as that run is rebuilt (`RescoredEntries.AddPostMaterialize`), +in the order the stage computed it. + Read the vocabulary this way: | | Visits | Holds | @@ -172,7 +191,7 @@ The pipeline is a fixed, four-element list, always in this order | `PerFileScoring` | 1-4 | fan-out (split 1) | 1..N | library + one run | | `FirstPassFDR` | 5 | **join** (barrier 1) | 1 | O(distinct entries) | | `PerFileRescoring` | 6 | fan-out (split 2) | 1..N | baseline + one run | -| `SecondPassFDR` | 7 | **join** (barrier 2) | 1 | O(survivors) | +| `SecondPassFDR` | 7 | **join** (barrier 2) | 1 | O(distinct entries) | Stages 1-4 are library preparation, mzML processing, calibration, and the main first-pass search that computes the 21 PIN features. Stage 5 is first-pass FDR plus the @@ -965,6 +984,13 @@ Experiment-wide: - `.1st-pass.model-diagnostics.json`, when `--model-diagnostics` is on - the pass-1 half of the report exists nowhere else by this point +A `SecondPassFDR` node rebuilds each run from these artifacts one at a time and drops it, so +it needs every run's files present but never holds more than one run's rows. The relay list +is therefore the whole cohort's, as it always was; what changed is the node's peak, not its +inputs. The run log says which shape it took - "folding over N run(s), each rebuilt from its +own artifacts and dropped" - and that line is the evidence, because a resident pool and a +fold produce identical output and differ only in a memory profile. + Not needed on the default path: `.1st-pass.fdr_scores.bin`. Establishing that is what issue #4486 was for - an orchestrator hands a `SecondPassFDR` node the per-run second-pass artifacts and the analysis-wide experiment sidecar, and nothing per-run from From 1cc6fcc13d443a544cc08ce1d3ef0f14446b1371 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 00:09:04 -0700 Subject: [PATCH 09/30] Pinned the per-file overlay chain's order and its refusal * The second-pass sidecar overlay writes an experiment q and the experiment-q floor raises it, so the two must run in the order they were added. Composing them the other way round applies a floor to a value about to be overwritten and reports q-values no run computed - a wrong number rather than a failure, which is why it needs a test and not a comment * Asserts the other half too: a milestone with no per-file source REFUSES an overlay rather than accepting one nothing will ever invoke. That is the silent case - the stage believes it applied something it did not * Covers MaterializeFile as well as StreamFiles, so the by-name accessor the streamed competition is driven through is not a second path with its own overlay semantics 606 tests (605 pass, 1 pre-existing skip). See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey.Test/ByproductContextTest.cs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs b/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs index 95f813841f..11027610c5 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ByproductContextTest.cs @@ -503,6 +503,62 @@ public void TestStreamFilesDropsEachFileAndRefusesALaterValueRead() Assert.ThrowsException(() => milestone.Value); } + /// + /// The per-file overlays run after the source, in the order they were added, on every + /// pass - and a milestone with no per-file source REFUSES one rather than accepting an + /// overlay nothing would ever invoke. + /// + /// Order is the correctness argument in Stage 7, not a detail: the second-pass + /// sidecar overlay writes an experiment q and the experiment-q floor then raises it, so + /// running them the other way round would apply a floor to a value about to be + /// overwritten and report q-values no run computed. Asserted by composing two overlays + /// that both append, and reading back the sequence. + /// + /// The refusal is the half that would otherwise fail silently. A resident run + /// stamps its entries in place and keeps them, so an overlay handed to it is simply + /// never called - the stage believes it applied something it did not, which is the + /// shape of defect this area has produced repeatedly. + /// + [TestMethod] + public void TestPostMaterializeOverlaysRunInOrderAndOnlyWithAPerFileSource() + { + var buffer = BufferWithFiles(@"file1", @"file2"); + var applied = new List(); + var milestone = new RescoredEntries(buffer, () => { }, + (fileName, entries) => applied.Add(fileName + @":source")); + milestone.AddPostMaterialize((fileName, entries) => applied.Add(fileName + @":first")); + milestone.AddPostMaterialize((fileName, entries) => applied.Add(fileName + @":second")); + + foreach (var kv in milestone.StreamFiles()) + Assert.IsNotNull(kv.Value); + CollectionAssert.AreEqual( + new[] + { + @"file1:source", @"file1:first", @"file1:second", + @"file2:source", @"file2:first", @"file2:second", + }, + applied, + @"Source then overlays, in the order added, for each file in turn"); + + // A second pass re-applies both, because the entries it applied to are gone. + applied.Clear(); + foreach (var kv in milestone.StreamFiles()) + Assert.IsNotNull(kv.Value); + Assert.AreEqual(6, applied.Count, @"Every pass re-applies the whole overlay chain"); + + // MaterializeFile - the by-name accessor the streamed competition uses - runs the + // same chain, so a consumer driven by the FDR layer's file order is not a second + // path with its own overlay semantics. + applied.Clear(); + milestone.MaterializeFile(@"file2"); + CollectionAssert.AreEqual( + new[] { @"file2:source", @"file2:first", @"file2:second" }, applied); + + var resident = new RescoredEntries(BufferWithFiles(@"file1")); + Assert.ThrowsException( + () => resident.AddPostMaterialize((fileName, entries) => { })); + } + private static List>> BufferWithOneFile() { return BufferWithFiles(@"file1"); From cb1d67369aa5fd5b136be3428ccd302dd078aeb3 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 02:03:13 -0700 Subject: [PATCH 10/30] Fixed what the review found: three ways the streamed join wrote or threw wrongly * CanStreamStage7Join now also requires protein-compact. Every other pass-2 mode still computes its per-file half in Stage 7, over the whole pool - RestorePass1Scalars, the resident second pass, the projection sink's protein-q map all index it. Streaming underneath them does not make them per-run, it takes their input away: the fragment release streams first and drops the pool, and ComputeAndPersist then throws hours into Stage 7. Same predicate ComputeAndPersist branches on for frozenCompetition, so the two move together when transfer's per-run half moves to the worker * ReadRetainedBaseIdsOrFail: the Stage 7 source THROWS instead of returning null. Returning null published a source-less milestone over the empty per-run lists the loader had already created, so the stage folded 446 empty runs, logged "No entries pass FDR threshold. Creating empty blib." and EXITED 0. The two sites decide on different evidence - the loader header-probes, this reads the body, minutes apart on a large cohort - so it is reachable with nothing wrong upstream. The sidecar's own reader documents absence as fatal * The resident 2nd-pass write loop is skipped on the streamed arm, and this is a correctness skip. It is reachable on the DEFAULT mode when nothing was recomputed, and a streamed run with a worker answer is rebuilt WITHOUT the 1st-pass overlay - that answer is where its scalars come from - so serializing those entries would overwrite every correct sidecar in the cohort with Score 0.0 and default experiment values. Skipping writes nothing that is missing: on this leg the file is PerFileRescoring's output and this task's input * haveSecondPass now reads the SAME evidence the fold uses - a PerFileRescoring validity stamp - instead of a format probe. They disagree exactly where it hurts: a sidecar written by a previous Stage 7 is format-current but stamped SecondPassFDR, so a probe said "skip the 1st-pass overlay" while the competition said "no worker answer, recompute", and it recomputed from blank rows. That is the OSPREY_STAGE7_STREAM=0 -> 1 A/B this branch adds * The per-run rebuild resolves the RECONCILED sibling. perFileParquetPaths holds whatever --input-scores named, and the documented fallback form names .scores.parquet - so rebuilding verbatim gave this arm pre-reconciliation rows while the resident arm on the same command line read the reconciled file. Two .blib files from one command line, and the byte-identity oracle could not see it because both arms were self-consistent * The 1st-pass experiment map is Lazy and goes through LoadPass1ExperimentRecords. On the default path RefillOneRunSurvivors never reads it, and it was ~400 MB resident for the whole stage; the raw ReadMap also returns null on an unreadable file, where the wrapper stops * StreamFiles marks _streamed BEFORE the yield. A consumer that breaks out of the walk left it false with runs already materialized, so a later Value read sailed past the guard and returned one populated run plus N-1 empty lists * regression.ps1 mode 10 SAVES and restores OSPREY_PASS2_QVALUE / OSPREY_EXPERIMENT_AGG instead of deleting them, and records a non-zero exit as a mode-10 failure. It sits before modes 4, 2, 5-9 and every later dataset, so a developer's exported arm was silently deleted and the rest of the suite ran the default while reporting the arm as passing * Restored the UTF-8 BOM that sed stripped from three files, and the three em dashes this branch introduced are ASCII hyphens 606 tests (605 pass, 1 pre-existing skip), inspection zero-warning. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.IO/FdrScoresSidecar.cs | 6 +- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 23 +++++- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 80 +++++++++++-------- .../Osprey/Osprey.Tasks/PipelineByproducts.cs | 7 +- .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 47 ++++++++++- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 2 +- pwiz_tools/Osprey/regression.ps1 | 24 +++++- 7 files changed, 144 insertions(+), 45 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs index b0744f2ef8..385ad36f84 100644 --- a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs @@ -629,7 +629,7 @@ public static bool TryRead(string path, IList entries, Pass expectedPa // Otherwise the sidecar carries an entry the caller's stub list // doesn't contain. The caller is expected to pass a SUPERSET of the // sidecar's entries (the post-rescore parquet for the 1st-pass - // sidecar, for example) — a record that fails to find its entry_id + // sidecar, for example) - a record that fails to find its entry_id // signals the sidecar was written from a different parquet (or from a // different binary version with different entry_id assignment). That // is corruption, not the gap-fill or post-compaction case we tolerate, @@ -770,7 +770,7 @@ private static bool TryWalkRecords(string path, Pass expectedPass, if (header[8] != FormatVersion) return false; // Reject mismatched pass bytes so a 2nd-pass sidecar can never - // be silently loaded into 1st-pass stubs (or vice versa) — the + // be silently loaded into 1st-pass stubs (or vice versa) - the // q-values would scramble without any visible error. if (header[9] != (byte)expectedPass) return false; @@ -778,7 +778,7 @@ private static bool TryWalkRecords(string path, Pass expectedPass, ulong headerCount = BitConverter.ToUInt64(header, 16); // Reject sidecars whose declared count exceeds physical // record capacity. (headerCount can validly be < the caller's - // entry count — see the remarks on the callers for the + // entry count - see the remarks on the callers for the // pre-gap-fill / post-compaction cases.) Use checked // arithmetic so a corrupt or malicious sidecar with a huge // headerCount is rejected loudly instead of wrapping int diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index b886763cb2..8ad1b4ed4b 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -453,7 +453,26 @@ void FlushPass2File(string fileName, IReadOnlyList records) // (#4486) both wrote the .bin + validity sidecar per file as they went, so // this loop is skipped for them - only the shared tallies they updated // drive the summary log below. - if (pass2Projections == null && !pass2SidecarsWritten) + // + // NOT ON THE STREAMED POOL, and this is a correctness skip rather than a + // memory one. It is reachable there on the DEFAULT mode: with + // recomputed == false - a cohort with no rescore work, or a resume whose + // sidecars are all current and worker-owned - the frozen competition inside + // `if (recomputed)` never ran, so pass2SidecarsWritten is false and this block + // would write. What it would write is the problem: a streamed run that already + // has a worker answer is rebuilt WITHOUT the 1st-pass overlay, because that + // answer is where its scalars come from, so its entries carry only the + // reconciled parquet's columns until the pass-2 overlay is installed - which + // happens after this method returns. Serializing them here would overwrite + // every correct sidecar in the cohort with Score 0.0 and default experiment + // values. + // + // Skipping writes nothing that is missing. On this leg the per-run 2nd-pass + // sidecar is PerFileRescoring's output and this task's INPUT - Outputs() says + // so - and "not recomputed" means every one of them is already current on + // disk. P13's never-conditionally-write rule binds the artifact's OWNER, and + // that is not this task here. + if (pass2Projections == null && !pass2SidecarsWritten && !rescored.Streams) { // Per-file progress: this writes one .2nd-pass.fdr_scores.bin per file // (~4.8 GB across 82) and was silent, which with the reload loop below is @@ -815,7 +834,7 @@ private static IReadOnlyDictionary LoadExperimentReco /// dependence in an artifact that is supposed to be route-independent is the exact class /// of defect mode 3 exists to catch, and it caught this one. /// - private static HashSet WorkerOwnedPass2Sidecars(PipelineContext ctx) + internal static HashSet WorkerOwnedPass2Sidecars(PipelineContext ctx) { var owned = new HashSet(StringComparer.Ordinal); if (ctx.Config?.InputFiles == null) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index e90e0b61cd..d8a2b479d7 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -2085,37 +2085,29 @@ private static Action> BuildStage7PerRunSource( { if (!ScoringTaskShared.CanStreamStage7Join(config)) return null; - var retainedBaseIds = ScoringTaskShared.ReadRetainedBaseIds(config, out _); - if (retainedBaseIds == null) - return null; - var experimentRecords = FdrExperimentSidecar.ReadMap( - FdrExperimentSidecar.PathFor(config.OutputBlib, - ScoringTaskShared.ArtifactSiblingPath(config), FdrScoresSidecar.Pass.FirstPass), - FdrScoresSidecar.Pass.FirstPass); + // THROWS rather than returning null. Returning null here would publish a + // source-less milestone over the empty per-run lists the loader has already + // created, and Stage 7 would fold 446 empty runs and write an empty .blib with + // exit 0. The two sites decide on different evidence - the loader header-probes, + // this reads the body, and they are minutes apart on a large cohort - so this is + // reachable without anything being wrong with the caller. + var retainedBaseIds = ScoringTaskShared.ReadRetainedBaseIdsOrFail(config); // Which runs already carry a second-pass answer, decided ONCE from the artifacts - // themselves (a header probe per run, no records read). A run in this set is - // rebuilt WITHOUT opening any first-pass file, which is the Boundary 3 -> 4 - // contract: its .2nd-pass.fdr_scores.bin holds every scalar the join reads, and the - // experiment-scope columns come from the analysis-wide sidecar. Probed here rather - // than per refill because each run is rebuilt once per fold pass, and the answer - // cannot change while the stage runs. - var inputByName = new Dictionary(StringComparer.Ordinal); - if (config.InputFiles != null) - { - foreach (string inputFile in config.InputFiles) - inputByName[Path.GetFileNameWithoutExtension(inputFile)] = inputFile; - } - var haveSecondPass = new HashSet(StringComparer.Ordinal); - foreach (var kv in perFileParquetPaths) - { - if (!inputByName.TryGetValue(kv.Key, out string inputFile)) - continue; - if (FdrScoresSidecar.IsCurrentFormat( - FdrScoresSidecar.Pass2Path(inputFile), FdrScoresSidecar.Pass.SecondPass)) - { - haveSecondPass.Add(kv.Key); - } - } + // themselves. A run in this set is rebuilt WITHOUT opening any first-pass file, + // which is the Boundary 3 -> 4 contract: its .2nd-pass.fdr_scores.bin holds every + // scalar the join reads, and the experiment-scope columns come from the + // analysis-wide sidecar. + // + // THE SAME EVIDENCE the fold itself uses - a validity stamp naming PerFileRescoring + // - and not a format probe. The two disagree exactly where it hurts: a sidecar + // written by a PREVIOUS Stage 7 (the OSPREY_STAGE7_STREAM=0 arm, say) is + // format-current but carries a SecondPassFDR stamp, so a probe would say "skip the + // first-pass overlay" while the competition, reading the stamp, says "no worker + // answer, recompute" - and it would then recompute from rows that carry only the + // reconciled parquet's columns, serializing Score 0.0 and default experiment values + // into every run's new sidecar. One question, one answer, one source. + var haveSecondPass = Pass2FdrSidecar.WorkerOwnedPass2Sidecars(ctx) + ?? new HashSet(StringComparer.Ordinal); // Says which shape Stage 7 took, for the reason its rescore sibling gives: without // it the only evidence is a memory profile, and "the gate is green so the new path // must have run" is the inference that lets a resident path pass as a streamed one. @@ -2128,6 +2120,17 @@ private static Action> BuildStage7PerRunSource( @"{2} of {0} run(s) carry a current 2nd-pass sidecar and are rebuilt without " + @"opening any 1st-pass file.", perFileParquetPaths.Count, retainedBaseIds.Count, haveSecondPass.Count)); + // LAZY, and deliberately so. On the default Boundary 3 -> 4 path every run carries a + // second-pass sidecar, RefillOneRunSurvivors dereferences this map only on the + // overlayFirstPass branch, and it is never read at all - while being ~400 MB + // resident for the whole of Stage 7 (6,044,771 records in the 446-run cohort's + // 266 MB sidecar), in the stage whose entire purpose is to stop holding things. + // Through LoadPass1ExperimentRecords, not FdrExperimentSidecar.ReadMap: the raw + // reader returns NULL on an unreadable file and the overlay then silently drops the + // experiment columns, where the wrapper stops. That is the same call this file + // already makes for the worker. + var experimentRecords = new Lazy>( + () => Pass2FdrSidecar.LoadPass1ExperimentRecords(config)); var sequencePool = ctx.Get().Value; return (fileName, survivors) => { @@ -2137,10 +2140,21 @@ private static Action> BuildStage7PerRunSource( @"Second-pass join hydrate: no scores parquet path published for {0}", fileName)); } - RescoreHydration.RefillOneRunSurvivors(fileName, parquetPath, survivors, - retainedBaseIds, experimentRecords, + // Resolve to the RECONCILED sibling, exactly as every other reader on this leg + // does. perFileParquetPaths holds whatever --input-scores named, and the + // documented fallback form names .scores.parquet - so rebuilding from the + // path verbatim would give this arm the PRE-reconciliation rows (first-pass + // boundaries, no Stage-6 gap-fill) while the resident arm on the same command + // line reads the reconciled file. Two different .blib files from one command + // line is precisely what the byte-identity oracle exists to prevent, and it + // would not have caught it: both arms would be self-consistent. + string effectivePath = + ParquetScoreCache.EffectiveScoresPathFromScoresPath(parquetPath); + bool overlayFirstPass = !haveSecondPass.Contains(fileName); + RescoreHydration.RefillOneRunSurvivors(fileName, effectivePath, survivors, + retainedBaseIds, overlayFirstPass ? experimentRecords.Value : null, (name, path) => ParquetScoreCache.LoadFdrStubsFromParquet(path, null, sequencePool), - !haveSecondPass.Contains(fileName)); + overlayFirstPass); }; } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs index 015b92704f..924f37cfd6 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PipelineByproducts.cs @@ -701,13 +701,18 @@ public IEnumerable>> StreamFiles(string labe foreach (var kv in base.Value) { progress?.Report(++done); + // Marked BEFORE the yield, not after. A consumer that breaks out of the + // walk - or an enumerator abandoned by an exception - would otherwise leave + // this false with runs already materialized, and a later Value read would + // sail past the guard and hand back one populated run plus N-1 empty lists: + // the silent almost-empty pool the guard exists to make impossible. + _streamed = true; _materializeFile(kv.Key, kv.Value); _postMaterialize?.Invoke(kv.Key, kv.Value); yield return kv; // Dropped as soon as the consumer's foreach body returns. TrimExcess too: // Clear leaves the backing array at its high-water capacity, which for a CHS // file is ~648 K references still committed per file. - _streamed = true; kv.Value.Clear(); kv.Value.TrimExcess(); } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index b007e20a63..a8ca5c689a 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -1,4 +1,4 @@ -/* +/* * Original author: Brendan MacLean , * MacCoss Lab, Department of Genome Sciences, UW * AI assistance: Claude Code (Claude Opus 4.8) @@ -473,8 +473,49 @@ internal static bool CanStreamStage7Join(OspreyConfig config) // by a longer route and with nothing in the log to say so. if (config.ModelDiagnostics) return false; - string path = RetainedBaseIdSidecar.PathFor(config.OutputBlib, ArtifactSiblingPath(config)); - return !string.IsNullOrEmpty(path) && RetainedBaseIdSidecar.IsCurrentFormat(path); + // And the pass-2 mode has to be the one whose per-run half already ran in the + // fan-out. protein-compact owns its whole per-file cycle in Stage 6 and Stage 7 + // folds the written answers; every other mode still computes the per-file half HERE, + // over the whole pool - RestorePass1Scalars, the resident second pass and the + // projection sink's per-file protein-q map all index it. Streaming underneath them + // does not make them per-run, it just takes their input away: the fragment release + // streams first and drops the pool, and ComputeAndPersist then throws + // "Value was read after StreamFiles dropped the survivor pool" hours into Stage 7. + // + // Not a guess about which modes are safe - the same predicate ComputeAndPersist + // itself branches on for `frozenCompetition`. When transfer's per-run half moves to + // Pass2PerFileWorker this term becomes "any mode with a worker" and the two move + // together. + if (!OspreyEnvironment.Pass2ProteinCompact) + return false; + string retainedPath = + RetainedBaseIdSidecar.PathFor(config.OutputBlib, ArtifactSiblingPath(config)); + return !string.IsNullOrEmpty(retainedPath) && + RetainedBaseIdSidecar.IsCurrentFormat(retainedPath); + } + + /// + /// The retained base_id set for the streamed second-pass join, or a hard failure. + /// + /// Separate from 's null-returning form because + /// the CALLER cannot degrade here. By the time Stage 7 asks, the --input-scores + /// load has already published one EMPTY list per run on the strength of + /// - which only header-probes the sidecar - so a null + /// leaves the stage folding over 446 empty runs, logging "No entries pass FDR threshold. + /// Creating empty blib." and exiting 0. An empty .blib from a successful-looking + /// run is the worst outcome this pipeline can produce, and the sidecar's own reader + /// documents its absence as FATAL. + /// + internal static HashSet ReadRetainedBaseIdsOrFail(OspreyConfig config) + { + var retained = ReadRetainedBaseIds(config, out string error); + if (retained != null) + return retained; + throw new InvalidDataException(string.Format( + @"The second-pass join is streaming, which requires the analysis-wide retained " + + @"base_id summary, and it could not be read: {0} Continuing would fold every run " + + @"as empty and write an empty library.", + error ?? @"(no reason reported)")); } /// diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index c241be1ecc..b38e6dcbdd 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -1,4 +1,4 @@ -/* +/* * Original author: Brendan MacLean , * MacCoss Lab, Department of Genome Sciences, UW * AI assistance: Claude Code (Claude Opus 5) diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index dac9e613d8..a9f4aacfbb 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -2097,13 +2097,33 @@ foreach ($name in $selected) { Write-Progress-Tc "${name}: $($arm.Tag) arm runs and produces (mode 10)" $altDir = Join-Path (Join-Path $runRoot $name) ("alt-" + $arm.Tag) $m10 = [pscustomobject]@{ Issues = [System.Collections.Generic.List[string]]::new() } - foreach ($k in $arm.Env.Keys) { Set-Item -Path "Env:$k" -Value $arm.Env[$k] } + # SAVE and RESTORE, not set-and-delete. Removing the variable does not put back a + # value the caller already had, and this mode sits BEFORE modes 4, 2, 5, 6, 7, 8 + # and 9 and before every later dataset - so a developer who exported + # OSPREY_PASS2_QVALUE=transfer and ran the suite had it silently deleted here, and + # every remaining leg ran the default while the summary reported them as passing + # the arm under test. The file's own convention is save-and-restore + # ($priorVerifyWorker, $script:priorAllowResident); this had missed it. + $priorArmEnv = @{} + foreach ($k in $arm.Env.Keys) { + $priorArmEnv[$k] = [Environment]::GetEnvironmentVariable($k) + Set-Item -Path "Env:$k" -Value $arm.Env[$k] + } try { $rAlt = Invoke-OspreyRun -Mzmls $inputs.Mzmls -Library $inputs.Library ` -Resolution $cfg.Resolution -WorkDir $altDir -LogName "alt-$($arm.Tag).log" ` -Spec $cfg -Manifest $inputs.Manifest + if ($rAlt.ExitCode -ne 0) { + $m10.Issues.Add("$($arm.Tag): Osprey exited $($rAlt.ExitCode) (see $($rAlt.Log))") + } } finally { - foreach ($k in $arm.Env.Keys) { Remove-Item -Path "Env:$k" -ErrorAction SilentlyContinue } + foreach ($k in $arm.Env.Keys) { + if ($null -eq $priorArmEnv[$k]) { + Remove-Item -Path "Env:$k" -ErrorAction SilentlyContinue + } else { + Set-Item -Path "Env:$k" -Value $priorArmEnv[$k] + } + } } $altBlib = Join-Path $altDir 'output.blib' if (-not (Test-Path $altBlib) -or (Get-Item $altBlib).Length -eq 0) { From ded647263e87f6ec8916405b3945d3aa32b88485 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 04:12:00 -0700 Subject: [PATCH 11/30] Made the gate able to see which Stage 7 shape ran, and stopped a partial sidecar walk * mode 3 now asserts the marker line for the per-run fold. A resident Stage 7 and a streamed one produce identical bytes BY DESIGN - that is the whole claim - so nothing this gate compares can tell them apart, and doc 00's rule for exactly that case is to assert the path rather than trust the bytes to reveal it. Without it, a change that silently disqualifies the streamed arm leaves every leg green while the O(runs x entries) peak returns * Scoped to the datasets that can stream, which is the finding as much as the fix: CanStreamStage7Join declines under --model-diagnostics, and this suite sets it on every dataset but plain Stellar - so three of four exercise only the resident arm. Recorded in the TODO rather than papered over by widening the assertion into a failure * FdrScoresSidecar.TryWalkRecords no longer returns false after handing records to its caller. Chunking the read (4b9df2a836) moved the file access INSIDE the walk, and with it the guarantee the OOM-filtered catch had rested on: File.ReadAllBytes and the header checks used to complete before any entry was touched, so `false` could only ever mean "nothing applied". A mid-walk IO fault now leaves half a file's values on the caller's entries, and OverlayPass2SidecarOntoFile's caller treats false as non-fatal ("protein FDR will use stale 1st-pass q-values") - shipping a run that is half pass-1 and half pass-2. It throws instead, naming the file and the count regression.ps1 -Dataset Stellar PASSED, 16 legs - the new one reads "mode3 (streamed join): PASS (per-run fold, no all-runs pool)". 606 tests, inspection zero-warning. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.IO/FdrScoresSidecar.cs | 39 ++++++++++++++++++- pwiz_tools/Osprey/regression.ps1 | 27 +++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs index 385ad36f84..3f74f22a63 100644 --- a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs @@ -745,6 +745,10 @@ public static bool TryReadOverlay(string path, private static bool TryWalkRecords(string path, Pass expectedPass, Func onRecord) { + // How many records the caller has already been handed. Decides whether a fault + // means "unusable file" (return false, nothing applied) or "partly applied" + // (throw) - see the catch below. + long delivered = 0; // NOT a bare catch: an OutOfMemoryException here is reported as a MISSING // sidecar, and a missing 1st-pass sidecar leaves those entries at Score 0.0. // The decoy side is not q-gated, so the zeros then compete in the picked- @@ -793,23 +797,56 @@ private static bool TryWalkRecords(string path, Pass expectedPass, { int take = Math.Min(RECORDS_PER_CHUNK, remaining); if (!ReadFully(fs, chunk, take * RecordLength)) - return false; + return delivered == 0 ? false : ThrowPartialWalk(path, delivered); remaining -= take; for (int rec = 0; rec < take; rec++) { if (!onRecord(chunk, rec * RecordLength)) return false; + delivered++; } } } } catch (Exception ex) when (!(ex is OutOfMemoryException)) { + // A failure BEFORE the first record is "this file is unusable", which every + // caller handles: it leaves their entries exactly as they arrived. A failure + // AFTER records have already been applied is a different fact and must not + // share the same answer - the caller has half a file's values on its entries + // and no way to know, and OverlayPass2SidecarOntoFile's caller treats false as + // non-fatal ("protein FDR will use stale 1st-pass q-values"), which would ship + // a run that is half pass-1 and half pass-2. + // + // This distinction was free before the read was chunked: File.ReadAllBytes and + // the header checks all completed before any entry was touched, so the catch + // could only ever mean "nothing applied". Chunking moved the read inside the + // walk and quietly took that guarantee away. + if (delivered > 0) + ThrowPartialWalk(path, delivered, ex); return false; } return true; } + /// + /// Report a sidecar walk that failed AFTER handing records to its caller. Always + /// throws; the bool return type only exists so the mid-loop call site can be an + /// expression. + /// + private static bool ThrowPartialWalk(string path, long delivered, Exception inner = null) + { + string message = string.Format( + @"Reading the FDR sidecar '{0}' failed after {1} record(s) had already been " + + @"applied. Those entries now hold this file's values and the rest do not, which " + + @"no caller can detect or undo, so the run stops here rather than continuing " + + @"with a partly-overlaid pool.", + path, delivered); + if (inner != null) + throw new IOException(message, inner); + throw new IOException(message); + } + /// /// Stream every record of a per-file sidecar to as a /// decoupled (entry_id + SVM score + 5 q-values + diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index a9f4aacfbb..92a001b77e 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -1974,6 +1974,33 @@ foreach ($name in $selected) { $summaryLines.Add("$name mode3 (shipped fold): PASS (worker answer folded for every file)") } + # Which SHAPE phase 4 folded in, asserted as a marker line rather than inferred from the + # output. A resident Stage 7 and a streamed one produce identical bytes by design - that + # is the whole claim - so nothing this gate compares can tell them apart, and doc 00's + # rule for exactly that situation is to assert the path the run reports rather than trust + # the bytes to reveal it. Without this, a change that silently disqualifies the streamed + # arm leaves every leg green while the O(runs x entries) peak comes back. + # + # Scoped to the datasets that can actually stream: CanStreamStage7Join declines under + # --model-diagnostics, which this suite sets on every dataset but plain Stellar, so + # demanding the line elsewhere would fail runs for a contract they cannot make. That + # narrowness is itself the finding - three of four datasets exercise only the resident + # arm - and it is recorded in the TODO rather than papered over here. + $chainCanStream = -not $cfg.ModelDiagnostics + $chainStreamed = Select-String -Path (Join-Path (Join-Path $chainRoot 'logs') 'phase4.log') ` + -Pattern 'Second-pass join: folding over \d+ run\(s\)' -Quiet + if (-not $chainCanStream) { + $summaryLines.Add("$name mode3 (streamed join): SKIP (--model-diagnostics keeps the resident pool)") + } elseif (-not $chainStreamed) { + $overallFail = $true + Write-Problem-Tc ("$name mode3 (streamed join): FAIL - phase 4 did not report the " + + "per-run fold, so SecondPassFDR built the whole-run survivor pool. Output is " + + "unchanged either way; only this line distinguishes them.") + $summaryLines.Add("$name mode3 (streamed join): FAIL") + } else { + $summaryLines.Add("$name mode3 (streamed join): PASS (per-run fold, no all-runs pool)") + } + # Scoped for the same reason as the shipped-fold check above: the verifier only exists on # the frozen-competition path, so OSPREY_PASS2_QVALUE=transfer and the retrain modes emit # NEITHER fold line and there is no split to assert. Detected from the straight leg having From 68cbcffd559bb3a39e215d20473a8523037b42ef Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 08:00:48 -0700 Subject: [PATCH 12/30] Cut mode 10 to one arm, and made it assert that the arm engaged * Dropped the standalone transfer arm. protein-compact REFUSES a mean(best-N) first pass, so the mean-best arm runs transfer for pass 2 regardless - one leg exercises both ideas, and the standalone arm bought a second run of the same pass-2 code while varying the first-pass aggregation away from the one that needs covering. Measured 223.1s of a 01:19:30 Perf/Regression wall that has to come in under 75 minutes. It stays reproducible by hand (OSPREY_PASS2_QVALUE=transfer, OSPREY_EXPERIMENT_AGG unset), which is what you want when the leg reds and you need to know which of the two ideas moved * The leg now asserts the arm ENGAGED before asserting anything about what it produced. It did not, and that was a hole in exactly the shape this leg exists to close: every file check in it passes on a run with both variables ignored, because the DEFAULT mode writes the same set of files. A green test for an arm that never ran is what let `transfer` reach production writing no experiment sidecar in the first place * Marker lines, not values, per doc 00's rule for a contract output cannot distinguish - the banners Program.cs and ComputeAndPersist already print. BOTH halves, because the arm is the pair: a run that took transfer but ignored OSPREY_EXPERIMENT_AGG would satisfy a one-line check and cover half of what this protects * Negative control, so the assertion is not vacuously true: neither marker appears in a default run's log (0 and 0 across the 446-run CHS run), which instead reports "Experiment aggregation: max (default - best observation per unit)" regression.ps1 -Dataset StellarLibDecoy PASSED, 22 legs, mode 10 green with both markers. The leg is now 216.9s where the two arms were 446.8s. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- pwiz_tools/Osprey/regression.ps1 | 46 ++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 92a001b77e..5f007dc9fa 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -2113,13 +2113,30 @@ foreach ($name in $selected) { # 82-file comparison wants re-running), and a golden would freeze a number nobody has # agreed on yet. What must not change silently is that the arm RUNS and PRODUCES. # - # The two arms pair as they must: protein-compact REFUSES a mean(best-N) first pass, so the - # mean-best leg necessarily runs transfer as its pass-2 mode. + # ONE arm, not two. protein-compact REFUSES a mean(best-N) first pass, so the mean-best arm + # necessarily runs transfer as its pass-2 mode - which means this single leg exercises both + # ideas, and a standalone transfer arm bought a second run of the same pass-2 code for the + # sake of varying the first-pass aggregation away from the one that needs covering. + # + # Measured, which is why it went: the two arms were 223.1 s and 223.7 s on StellarLibDecoy, + # 7.4 min together, against a Perf/Regression wall of 01:19:30 that has to come in under + # 75 minutes. The standalone arm is the half whose coverage is already implied. + # + # It stays reproducible by hand for diagnosis - that is its remaining value, and it is a + # one-line env var: OSPREY_PASS2_QVALUE=transfer with OSPREY_EXPERIMENT_AGG unset. Isolating + # transfer from mean-best-N is what you want when this leg goes red and you need to know + # which of the two moved; it is not what you want on every gate run. if (-not $SkipAltPass2 -and $cfg.AltPass2) { + # Markers are the banners Osprey prints for each arm - Program.cs's + # DescribeExperimentAgg, and ComputeAndPersist's OSPREY_PASS2_QVALUE line. Regexes, so + # the surrounding prose can change without breaking the gate; what they pin is the + # mode NAME and, for mean-best, the word ACTIVE that only the engaged path emits. $altArms = @( - @{ Tag = 'transfer'; Env = @{ OSPREY_PASS2_QVALUE = 'transfer' } }, - @{ Tag = 'meanbest2'; Env = @{ OSPREY_PASS2_QVALUE = 'transfer' - OSPREY_EXPERIMENT_AGG = 'mean-best-2' } }) + @{ Tag = 'meanbest2' + Env = @{ OSPREY_PASS2_QVALUE = 'transfer' + OSPREY_EXPERIMENT_AGG = 'mean-best-2' } + Markers = @('OSPREY_PASS2_QVALUE=transfer:', + 'Experiment aggregation: mean-best-2 ACTIVE') }) foreach ($arm in $altArms) { Write-Progress-Tc "${name}: $($arm.Tag) arm runs and produces (mode 10)" $altDir = Join-Path (Join-Path $runRoot $name) ("alt-" + $arm.Tag) @@ -2152,6 +2169,25 @@ foreach ($name in $selected) { } } } + # THE ARM ACTUALLY RAN, asserted from the banner each mode prints, before anything + # about what it produced. Without this the leg is green for an arm that never + # engaged: every check below passes on a run with both variables ignored, because + # the DEFAULT mode writes the same set of files. That is not a hypothetical failure + # shape - it is this leg's own reason for existing, since `transfer` reached + # production writing no experiment sidecar precisely because nothing ran it. + # + # A marker line rather than a value comparison, per doc 00: where a contract cannot + # be distinguished by output, the gate must assert the path the run reports. Both + # halves are checked because the arm is the PAIR - a run that took `transfer` but + # ignored OSPREY_EXPERIMENT_AGG would satisfy a one-line check and cover only half + # of what this leg is here to protect. + foreach ($marker in $arm.Markers) { + if (-not (Select-String -Path $rAlt.Log -Pattern $marker -Quiet)) { + $m10.Issues.Add(("$($arm.Tag): the run log does not report /$marker/, so the " + + "arm did not engage and every check below would pass on a " + + "default run")) + } + } $altBlib = Join-Path $altDir 'output.blib' if (-not (Test-Path $altBlib) -or (Get-Item $altBlib).Length -eq 0) { $m10.Issues.Add("$($arm.Tag): no output.blib written") From a9f5190b8f5f16f636f749c03c125a017a91a4e0 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 15:12:53 -0700 Subject: [PATCH 13/30] Changed pass-2 model diagnostics to fold run by run * Added ModelDiagnosticsData.Accumulator.BuildPass2, so SecondPassFDR builds the report from folded reductions instead of the resident survivor pool * Split the co-assignment panel into caller-driven phases so its cutoff pass rides along with the fold, costing two stream passes rather than three * Removed the ModelDiagnostics term from CanStreamStage7Join, so mode 3 now exercises the streamed join on all four datasets instead of one See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../ModelDiagnosticsData.Accumulator.cs | 121 +++++++++++- .../ModelDiagnosticsData.CoAssignment.cs | 142 +++++++++++-- .../ModelDiagnosticsReport.cs | 101 ++++++++-- .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 20 +- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 122 ++++++++++-- .../Osprey.Test/ModelDiagnosticsDataTest.cs | 187 ++++++++++++++++++ .../Osprey/docs/00-pipeline-architecture.md | 9 + pwiz_tools/Osprey/regression.ps1 | 66 +++++-- 8 files changed, 692 insertions(+), 76 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.Accumulator.cs b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.Accumulator.cs index b04d37ca57..1c75a6589f 100644 --- a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.Accumulator.cs +++ b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.Accumulator.cs @@ -73,6 +73,13 @@ public sealed class Accumulator private readonly string[] _runNames; private readonly int _nFiles; + // 1 = pre-compaction first pass, 2 = final reported pool. The reductions are the + // same either way; this decides which ones are folded at all and which Build method + // may be called. The Frontier is the one pass-1-only fold, and skipping it in pass 2 + // is not a micro-optimisation: it is a dictionary lookup and update per target row + // over the whole reported pool, and nothing in Pass2Data reads the result. + private readonly int _pass; + // Best-per-precursor, keyed modseq|charge (== ReduceToPrecs). private readonly Dictionary _best = new Dictionary(StringComparer.Ordinal); @@ -124,14 +131,20 @@ public sealed class Accumulator /// entrapment-to-target DB ratio r. /// configured run-level FDR. /// reported FDR control level (drives EffectiveRunQvalue). + /// 1 for the pre-compaction first pass (), + /// 2 for the final reported pool (). Decides which folds run. public Accumulator( string[] runNames, IReadOnlyDictionary classByBaseId, IReadOnlyDictionary pairByBaseId, double entrapmentRatio, double runFdr, - FdrLevel fdrLevel) + FdrLevel fdrLevel, + int pass = 1) { + if (pass != 1 && pass != 2) + throw new ArgumentOutOfRangeException(nameof(pass)); + _pass = pass; _runNames = runNames ?? throw new ArgumentNullException(nameof(runNames)); _nFiles = runNames.Length; _classByBaseId = classByBaseId; @@ -248,8 +261,9 @@ public void Add(int fileIdx, string modifiedSequence, byte charge, uint entryId, // Frontier: fold the UN-GATED first-pass row into the within-file run-q tally // (target side only). On a new file, flush the previous file's per-precursor best - // run-q into the bins first (rows arrive in file-major order). - if (!isDecoy) + // run-q into the bins first (rows arrive in file-major order). Pass 1 only - + // Pass2Data has no Frontier card, so in pass 2 this is work with no reader. + if (_pass == 1 && !isDecoy) { if (fileIdx != _frontierCurFile) { @@ -288,6 +302,8 @@ public void Add(int fileIdx, string modifiedSequence, byte charge, uint entryId, /// public ModelDiagnosticsData Build(FeatureContributions contributions) { + if (_pass != 1) + throw new InvalidOperationException(PassMismatch(1, nameof(BuildPass2))); var precs = _best.Values.ToList(); var data = new ModelDiagnosticsData { @@ -369,6 +385,105 @@ public ModelDiagnosticsData Build(FeatureContributions contributions) return data; } + /// + /// Assemble the pass-2 from the accumulated reductions, + /// running the SAME downstream builders the batch uses over + /// the resident pool. This is the streamed half of the fix for the last + /// O(runs x entries) structure in Stage 7: the batch method takes + /// IReadOnlyList<KeyValuePair<string, List<FdrEntry>>> and + /// means it, so --model-diagnostics forced SecondPassFDR to hold every run's + /// survivors resident and CanStreamStage7Join declined the streamed join + /// outright whenever the report was asked for. + /// + /// Eight of the nine pass-2 cards are reductions this accumulator already + /// holds - best-per-precursor, per-file passing counts, cross-run membership and the + /// per-base_id win-fraction maxima - so they cost a walk of the O(distinct) reduced + /// state here rather than a walk of the pool. The ninth, + /// , is NOT foldable in one pass: its acceptance + /// boundary is a reduction over every row that its per-row verdicts are then compared + /// against, so it needs the pool twice. It is therefore built by the CALLER, from a + /// second stream pass, and passed in - the same division pass 1 makes, where the + /// panel comes from PeakCoAssignmentSource rather than from the fold. + /// + /// No here, deliberately, where the batch + /// BuildPass2 carries one per card (#4571). There the cards WERE the expensive part - + /// six independent whole-pool walks. Here the pool walk has already happened in the + /// caller's stream, which reports per run, and what is left walks only the reduced + /// state. A card reporter would print six lines inside one second, on every run + /// forever. + /// + /// The retrained second-pass model, or null under + /// confidence-transfer mode, which leaves the structural half null exactly as the + /// batch path does. + /// The pass-2 co-assignment panel built from the caller's + /// second stream pass; null leaves the panel out. + public Pass2Data BuildPass2(FeatureContributions contributions, + CoAssignmentData coAssignment) + { + if (_pass != 2) + throw new InvalidOperationException(PassMismatch(2, nameof(Build))); + var precs = _best.Values.ToList(); + var pass2 = new Pass2Data(); + + // Q-driven half: available whenever a second pass produced reported q-values + // (retrain OR confidence transfer). Entrapment-independent except FdpViews. + var perFile = new List(_nFiles); + for (int f = 0; f < _nFiles; f++) + { + perFile.Add(new FileSummaryRow + { + File = _runNames[f], + Targets = _fileTargets[f], + Decoys = _fileDecoys[f], + Entrapment = _fileEntrap[f], + }); + } + pass2.PerFile = perFile; + pass2.IdYield = BuildIdYield(precs); + + double r = _entrapmentRatio > 0 ? _entrapmentRatio : 1.0; + // Close the run in progress and any trailing runs that contributed nothing, so + // every file index has its entry - the same obligation Build has, for the same + // reason, and the one place a streamed reduction can silently disagree with the + // batch one. + _runStream.Finish(); + _expStream.Finish(); + _entRunStream.Finish(); + _entExpStream.Finish(); + pass2.CrossRun = new CrossRunDetection + { + RunNames = _runNames, + PerRun = ComputeCrossRunView(_runStream, _anyEntrapment ? _entRunStream : null, _nFiles, r), + Experiment = ComputeCrossRunView(_expStream, _anyEntrapment ? _entExpStream : null, _nFiles, r), + }; + + pass2.FdpViews = BuildPass2FdpViews(precs, _entrapmentRatio); + pass2.CoAssignment = coAssignment; + + // Structural half: only when the second pass retrained on the reported pool. + // Null contributions (transfer mode) leave Model, DensityRatio and WinFraction + // null and the report's structural cards show their n/a note. + pass2.Model = BuildModelPass2(contributions, precs); + if (pass2.Model != null) + { + bool hasEntrapment = precs.Any(p => p.Class == EntrapmentClass.PTarget); + pass2.DensityRatio = BuildDensityRatio(pass2.Model.Scores, hasEntrapment); + pass2.WinFraction = BuildWinFractionFromReduced(_bt, _tClass); + } + return pass2; + } + + // Both Build methods read reductions that only their own pass folds, so calling the + // wrong one returns a plausible-looking object built from partly unfolded state + // rather than failing. Name the other method: the caller's mistake is always that + // the pass argument and the Build call disagree. + private string PassMismatch(int expected, string otherMethod) + { + return string.Format( + @"ModelDiagnosticsData.Accumulator was constructed for pass {0} but built for pass {1}. Use {2} instead, or construct it with pass: {1}.", + _pass, expected, otherMethod); + } + /// /// One cross-run membership reduction, folded run by run instead of retained run by /// run. Replaces a List<HashSet<string>> of N per-run key sets with diff --git a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs index ae2efb90dd..cebf237001 100644 --- a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs +++ b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs @@ -388,29 +388,100 @@ public static CoAssignmentData BuildCoAssignment( { if (perFileEntries == null || precursorMzByEntryId == null) return null; - bool haveManifest = classByBaseId != null && classByBaseId.Count > 0; var runNames = new string[perFileEntries.Count]; for (int f = 0; f < perFileEntries.Count; f++) runNames[f] = perFileEntries[f].Key; + return BuildCoAssignmentCore(runNames, () => perFileEntries, classByBaseId, + precursorMzByEntryId, runFdr, fdrLevel, pass, postReconciliation, stratumBaseIds); + } + private static CoAssignmentData BuildCoAssignmentCore( + string[] runNames, + Func>>> openStream, + IReadOnlyDictionary classByBaseId, + Func precursorMzByEntryId, + double runFdr, + FdrLevel fdrLevel, + int pass, + bool postReconciliation, + HashSet stratumBaseIds) + { var builder = new CoAssignmentPassBuilder(runNames, pass, postReconciliation, stratumBaseIds); // Phase 1: the decoy score cutoffs, over every row at every file. Cheap - no identity // string, no library lookup - so it can walk the whole pool. - for (int f = 0; f < perFileEntries.Count; f++) + int f1 = 0; + foreach (var kvp in openStream()) { - foreach (var e in perFileEntries[f].Value) - { - int wc0 = 0, woc0 = 0; - builder.ObserveCutoff(f, - Classify(e.IsDecoy, e.EntryId & BASE_ID_MASK, classByBaseId, haveManifest, ref wc0, ref woc0), - e.EntryId, e.Score, e.ExperimentAggregateScore, e.EffectiveRunQvalue(fdrLevel), - e.EffectiveExperimentQvalue(fdrLevel), runFdr); - } - // Reduce this file's bests to its cutoff before reading the next, so the - // builder never holds more than one file's worth. - builder.SealRunCutoff(f); + VerifyRunOrder(runNames, f1, kvp.Key); + ObserveCoAssignmentRun(builder, f1, kvp.Value, classByBaseId, runFdr, fdrLevel); + f1++; + } + VerifyRunCount(runNames, f1); + return BuildCoAssignmentDetection(builder, runNames, openStream(), classByBaseId, + precursorMzByEntryId, runFdr, fdrLevel); + } + + /// + /// Phase 1 of the co-assignment panel for ONE run: fold its rows into the acceptance + /// boundary and seal that run's cutoff. Public so a caller already streaming the pool for + /// another reason can fold this into the pass it is making rather than opening a third - + /// which is what the streamed second-pass join does, sharing this phase with the + /// diagnostics accumulator's fold. + /// + /// Sealing per run is what keeps the builder holding one run's bests rather than + /// the pool's, and it is why the caller must present runs in input order and present all + /// of them: the boundary phase 2 compares against is indexed by run position. + /// + public static void ObserveCoAssignmentRun( + CoAssignmentPassBuilder builder, + int fileIdx, + IEnumerable rows, + IReadOnlyDictionary classByBaseId, + double runFdr, + FdrLevel fdrLevel) + { + bool haveManifest = classByBaseId != null && classByBaseId.Count > 0; + foreach (var e in rows) + { + int wc0 = 0, woc0 = 0; + builder.ObserveCutoff(fileIdx, + Classify(e.IsDecoy, e.EntryId & BASE_ID_MASK, classByBaseId, haveManifest, ref wc0, ref woc0), + e.EntryId, e.Score, e.ExperimentAggregateScore, e.EffectiveRunQvalue(fdrLevel), + e.EffectiveExperimentQvalue(fdrLevel), runFdr); } + // Reduce this file's bests to its cutoff before reading the next, so the + // builder never holds more than one file's worth. + builder.SealRunCutoff(fileIdx); + } + + /// + /// Phase 2 of the co-assignment panel: seal the boundary phase 1 folded, walk the pool a + /// second time judging each row against it, and build the panel. Public for the same + /// reason is - the streamed second-pass join drives + /// the two phases from its own reads. + /// + /// Two reads and not one because the boundary is a reduction over every row that + /// every row is then compared against; there is no fold that yields both in one walk. + /// That is what makes this the one pass-2 card the diagnostics accumulator cannot absorb. + /// The alternative - reconstructing the reported pool from the per-file sidecars, as the + /// pass-1 panel does - would put the definition of "the reported pool" in a second place, + /// and that pool is defined by the rebuild the stream performs: retained base_ids, the + /// pass-2 sidecar overlay and the experiment-q floors. One definition, two reads. + /// + public static CoAssignmentData BuildCoAssignmentDetection( + CoAssignmentPassBuilder builder, + string[] runNames, + IEnumerable>> rows, + IReadOnlyDictionary classByBaseId, + Func precursorMzByEntryId, + double runFdr, + FdrLevel fdrLevel) + { + if (precursorMzByEntryId == null) + return null; + bool haveManifest = classByBaseId != null && classByBaseId.Count > 0; + int pass = builder.Pass; builder.SealCutoffs(); // Phase 2: the detected rows. @@ -433,9 +504,11 @@ public static CoAssignmentData BuildCoAssignment( // moves a count is answered here and nowhere upstream. using (var rowDump = FdrDiagnostics.CreateCoAssignRowDump(pass)) { - for (int f = 0; f < perFileEntries.Count; f++) + int f = 0; + foreach (var kvp in rows) { - foreach (var e in perFileEntries[f].Value) + VerifyRunOrder(runNames, f, kvp.Key); + foreach (var e in kvp.Value) { double runQ = e.EffectiveRunQvalue(fdrLevel); double expQ = e.EffectiveExperimentQvalue(fdrLevel); @@ -448,7 +521,7 @@ public static CoAssignmentData BuildCoAssignment( // Dumped for EVERY row, including the excluded ones. An entry that stopped // being counted and one that never was are the same absence in the panel's // output and different rows here, which is the distinction the A/B needs. - rowDump?.WriteRow(f, perFileEntries[f].Key, e.EntryId, e.EntryId & BASE_ID_MASK, + rowDump?.WriteRow(f, kvp.Key, e.EntryId, e.EntryId & BASE_ID_MASK, e.IsDecoy, cls.ToString(), e.Score, e.ExperimentAggregateScore, runQ, expQ, e.ApexRt, e.Charge, included, e.ModifiedSequence); if (!included) @@ -476,7 +549,9 @@ public static CoAssignmentData BuildCoAssignment( runQ, expQ, runFdr); } builder.FlushFile(); + f++; } + VerifyRunCount(runNames, f); // The boundaries every per-row verdict above was compared against. Written from // the builder rather than recomputed, for the same reason the verdict is. rowDump?.WriteCutoffs(builder.RunCutoff, runNames, builder.ExperimentCutoff, @@ -489,6 +564,37 @@ public static CoAssignmentData BuildCoAssignment( return anyMz && anyDistinctApexRt ? builder.Build() : null; } + /// + /// Assert that the run arriving at index is the run the panel + /// believes sits there. The two phases index the SAME builder state by position - phase 2 + /// compares each row against the per-run cutoff phase 1 sealed at that index - so a source + /// that yielded runs in a different order on the second pass would judge every row against + /// another run's boundary and still produce a complete, plausible panel. Nothing + /// downstream could detect it, which is why this throws rather than logs. + /// + private static void VerifyRunOrder(string[] runNames, int index, string runName) + { + if (index < runNames.Length && Equals(runNames[index], runName)) + return; + throw new InvalidOperationException(string.Format( + @"Peak co-assignment read run '{0}' at index {1}, where '{2}' was expected. The two phases index the acceptance boundary by run position, so the source must yield the same runs in the same order on both passes.", + runName, index, index < runNames.Length ? runNames[index] : @"(past the end)")); + } + + /// + /// Assert that a phase saw every run. A stream that ended early leaves the trailing runs + /// with no cutoff and no rows, which reads as "those runs identified nothing" rather than + /// as a truncated read - the same silent-shrink failure the pass-1 sidecar walk refuses. + /// + private static void VerifyRunCount(string[] runNames, int seen) + { + if (seen == runNames.Length) + return; + throw new InvalidOperationException(string.Format( + @"Peak co-assignment walked {0} run(s) where {1} were expected. A short read would report the missing runs as having identified nothing.", + seen, runNames.Length)); + } + /// /// Assembles one pass's by driving a /// per q scope, so both definitions of "detected" - @@ -526,6 +632,10 @@ public CoAssignmentPassBuilder(string[] runNames, int pass, bool postReconciliat _stratumBaseIds = stratumBaseIds; } + /// 1 = pre-compaction first-pass detection, 2 = final reported pool. Read by + /// the phase-2 driver, which tags the row dump with it. + public int Pass => _pass; + private readonly HashSet _stratumBaseIds; // Score at or above which a DECOY counts as detected: the worst score among the diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs b/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs index 5c79c9619d..c6d9bf9d9b 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs @@ -226,12 +226,9 @@ public static void WritePass2AndFinalize( { try { - var data = ReadJson(ResolvePass1SidecarPath(config)); + var data = ReadPass1ForEnrichment(config, logInfo); if (data == null) - { - logInfo(@"[MODEL-DIAGNOSTICS] pass-1 data sidecar not found; pass-2 enrichment skipped (pass-1 page stands)."); return; - } Dictionary classByBaseId; Dictionary pairByBaseId; @@ -258,18 +255,7 @@ public static void WritePass2AndFinalize( entrapmentRatio, config.RunFdr, config.FdrLevel, BuildPrecursorMzLookup(libraryById), stratumBaseIds); - // Pass 2's own product, written before the page for the same reason pass 1's is. - // NOTHING is deleted here and pass1.json is not rewritten: the two files are - // immutable products of different phases, and the page below is the only thing - // this method overwrites. - string pass2Path = ResolvePass2SidecarPath(config); - WriteJson(pass2Path, data.Pass2); - StampProduct(pass2Path, SecondPassTaskName, validityKey, logInfo); - string outPath = RenderAndWrite(data, config); - int pass2ViewCount = data.Pass2?.FdpViews?.Count ?? 0; - logInfo(string.Format( - @"[MODEL-DIAGNOSTICS] finalized report ({0} pass-2 FDR view(s); pass-2 model {1}); re-wrote: {2}", - pass2ViewCount, data.Pass2?.Model != null ? @"included" : @"n/a", outPath)); + FinalizePass2(data, config, validityKey, logInfo); } catch (Exception ex) { @@ -277,6 +263,89 @@ public static void WritePass2AndFinalize( } } + /// + /// End-of-run enrichment from the STREAMED pass-2 fold, the accumulator sibling of + /// . Same product, same page, same log line - the only + /// difference is that the pass-2 cards come from reductions folded run by run through + /// instead of from the resident survivor + /// pool, which is what lets CanStreamStage7Join stop declining the streamed join + /// whenever --model-diagnostics is asked for. + /// + /// Byte-identical to on the same input for the + /// reason the pass-1 pair already is: every reduction the accumulator performs is + /// order-independent, and both paths enumerate the reduced best-per-precursor set in the + /// same nested (file, row) order. The classification / pairing / ratio are not rebuilt + /// here - the accumulator was constructed with them, and rebuilding runs for minutes at + /// 6.3M library entries. + /// + /// is built by the caller from its own second + /// stream pass, because that panel's acceptance boundary is a reduction over every row + /// that its per-row verdicts are then compared against - it cannot be folded in the pass + /// that computes it. Null leaves the panel out, exactly as a null library lookup does on + /// the resident path. + /// + public static void WritePass2AndFinalizeFromAccumulator( + ModelDiagnosticsData.Accumulator accumulator, + ModelDiagnosticsData.CoAssignmentData coAssignment, + FeatureContributions pass2Contributions, + OspreyConfig config, + Action logInfo, + string validityKey = null) + { + try + { + var data = ReadPass1ForEnrichment(config, logInfo); + if (data == null) + return; + data.Pass2 = accumulator.BuildPass2(pass2Contributions, coAssignment); + FinalizePass2(data, config, validityKey, logInfo); + } + catch (Exception ex) + { + logInfo(string.Format(@"[MODEL-DIAGNOSTICS] pass-2 enrichment failed: {0}", ex.Message)); + } + } + + /// + /// The pass-1 data sidecar both enrichment paths append to, or null with the log line + /// explaining that the pass-1 page stands unchanged. Absence is a degrade, not a failure: + /// pass 1's page is a complete statement of the first pass on its own. + /// + private static ModelDiagnosticsData ReadPass1ForEnrichment(OspreyConfig config, + Action logInfo) + { + var data = ReadJson(ResolvePass1SidecarPath(config)); + if (data == null) + { + logInfo(@"[MODEL-DIAGNOSTICS] pass-1 data sidecar not found; pass-2 enrichment skipped (pass-1 page stands)."); + return null; + } + return data; + } + + /// + /// Write pass 2's own product and re-render the page from the enriched graph, shared by + /// the resident and streamed enrichment paths so the two cannot drift in what they emit. + /// + /// The product goes down before the page for the same reason pass 1's does: an + /// interruption between the two leaves the artifact that can rebuild the view rather than + /// a view with nothing behind it. NOTHING is deleted here and pass1.json is not + /// rewritten - the two files are immutable products of different phases, and the page is + /// the only thing this overwrites. + /// + private static void FinalizePass2(ModelDiagnosticsData data, OspreyConfig config, + string validityKey, Action logInfo) + { + string pass2Path = ResolvePass2SidecarPath(config); + WriteJson(pass2Path, data.Pass2); + StampProduct(pass2Path, SecondPassTaskName, validityKey, logInfo); + string outPath = RenderAndWrite(data, config); + int pass2ViewCount = data.Pass2?.FdpViews?.Count ?? 0; + logInfo(string.Format( + @"[MODEL-DIAGNOSTICS] finalized report ({0} pass-2 FDR view(s); pass-2 model {1}); re-wrote: {2}", + pass2ViewCount, data.Pass2?.Model != null ? @"included" : @"n/a", outPath)); + } + /// /// Re-render the page from the diagnostics PRODUCTS already on disk, doing no analysis /// of any kind: read 1st-pass.model-diagnostics.json, attach 2nd-pass.model-diagnostics.json if it is there, render. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index a8ca5c689a..61718b09ce 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -464,16 +464,16 @@ internal static bool CanStreamStage7Join(OspreyConfig config) return false; if (PerFileScoringTask.NeedsResidentPool(config, OspreyEnvironment.UseFdrProjection)) return false; - // --model-diagnostics is the fourth requirement, and it is a CURRENT limitation - // rather than a property of the leg. The pass-2 report builders index their files by - // position and revisit a file across two loops, so they need a list and not a - // stream; the fold that removes the need is the accumulator the pass-1 report - // already uses. Declining here rather than letting the report leg stream and then - // silently pull the whole pool back through .Value, which is the same peak reached - // by a longer route and with nothing in the log to say so. - if (config.ModelDiagnostics) - return false; - // And the pass-2 mode has to be the one whose per-run half already ran in the + // --model-diagnostics WAS the fourth requirement, and is no longer one. The pass-2 + // report is now folded run by run through ModelDiagnosticsData.Accumulator - the + // same accumulator the pass-1 report uses - with the co-assignment panel's two + // phases driven from the join's own stream passes, so the report no longer needs a + // list it can index by position. Removing this term is also what lets the gate SEE + // the streamed arm: --model-diagnostics is set on StellarLibDecoy, + // StellarGenDecoyEntrap and Astral, so while it stood here mode 3's phase 4 took the + // resident path on three of the four datasets and a streamed-arm defect needing + // library decoys, entrapment or hram data passed the suite green. + // The pass-2 mode has to be the one whose per-run half already ran in the // fan-out. protein-compact owns its whole per-file cycle in Stage 6 and Stage 7 // folds the written answers; every other mode still computes the per-file half HERE, // over the whole pool - RestorePass1Scalars, the resident second pass and the diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index b38e6dcbdd..5fa007a0f1 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -27,6 +27,7 @@ using System.IO; using pwiz.Osprey.Core; using pwiz.Osprey.FDR; +using pwiz.Osprey.FDR.ModelDiagnostics; using pwiz.Osprey.IO; using pwiz.Osprey.Tasks.ModelDiagnostics; @@ -501,23 +502,120 @@ public override bool Run(PipelineContext ctx) if (OspreyEnvironment.Pass2ProteinCompact && ctx.TryGet(out var pcStratum)) stratumBaseIds = pcStratum.BaseIds; - // The ONE Stage 7 consumer still handed the whole pool, and the only one that - // cannot simply be re-pointed at the stream: the pass-2 data builders index - // their files BY POSITION and revisit a file across two loops - // (ModelDiagnosticsData.CoAssignment), so an enumerable that rebuilds a run per - // pass would rebuild the same run several times per view. The fold it wants is - // the one FirstPassFDR's pass-1 accumulator already is, and giving pass 2 the - // same accumulator is the fix - not a wider parameter type. Until then - // CanStreamStage7Join declines the report leg outright, so reaching this line - // means the pool is resident and .Value is what it has always been. - ModelDiagnosticsReport.WritePass2AndFinalize( - rescored.Value, pass2Contributions, libraryById, config, ctx.LogInfo, - stratumBaseIds, ValidityKey(ctx)); + // Two shapes, one report. The streamed arm folds the run-by-run reductions the + // pass-2 cards are made of instead of holding every run's survivors, which is + // what removed the last O(runs x entries) structure in Stage 7 and let + // CanStreamStage7Join stop declining this leg outright. The resident arm is + // unchanged and is the A/B oracle the streamed one is verified against: same + // page, same 2nd-pass.model-diagnostics.json. + if (rescored.Streams) + { + WritePass2DiagnosticsStreamed(ctx, rescored, pass2Contributions, libraryById, + config, stratumBaseIds); + } + else + { + ModelDiagnosticsReport.WritePass2AndFinalize( + rescored.Value, pass2Contributions, libraryById, config, ctx.LogInfo, + stratumBaseIds, ValidityKey(ctx)); + } } return true; } + /// + /// Build and write the pass-2 --model-diagnostics product from the STREAMED + /// survivor source, holding no more than one run's entries at a time. + /// + /// Two passes over the stream, and the split between them is forced rather than + /// chosen. Eight of the nine pass-2 cards are reductions + /// already folds per row, so they ride + /// along in the first pass for free. The ninth - peak co-assignment - draws an acceptance + /// boundary that is itself a reduction over every row, and then compares every row against + /// it, so its own phase 1 joins the first pass and only its phase 2 needs a second read. + /// Two passes, not three. + /// + /// Wall clock is the cost, and it is the axis this work is allowed to spend: each + /// pass rebuilds every run from its own artifacts. Stage 7 already re-streams several + /// times over (the experiment-q reclamp, the retained base_ids, protein FDR, the FDRBench + /// TSV and the blib gates), so this is the stage's existing idiom rather than a new cost + /// class - and it replaces a 78.3 GB resident pool. + /// + /// Guarded here rather than only inside the report writer, because the fold runs + /// OUTSIDE it: + /// catches its own work, but the two stream passes and the classification happen before + /// it is called, and the co-assignment order guards throw BY DESIGN. A diagnostics-only + /// artifact must never take down a ten-hour search - the same reason + /// PeakCoAssignmentSource.Build wraps itself for its pass-1 callers. Refusing the + /// panel and logging why is the intended outcome of those guards; losing the run is not. + /// + private void WritePass2DiagnosticsStreamed(PipelineContext ctx, RescoredEntries rescored, + FeatureContributions pass2Contributions, + IReadOnlyDictionary libraryById, + OspreyConfig config, HashSet stratumBaseIds) + { + try + { + WritePass2DiagnosticsStreamedCore(ctx, rescored, pass2Contributions, libraryById, + config, stratumBaseIds); + } + catch (Exception ex) + { + ctx.LogInfo(string.Format( + @"[MODEL-DIAGNOSTICS] pass-2 enrichment failed: {0}", ex.Message)); + } + } + + private void WritePass2DiagnosticsStreamedCore(PipelineContext ctx, RescoredEntries rescored, + FeatureContributions pass2Contributions, + IReadOnlyDictionary libraryById, + OspreyConfig config, HashSet stratumBaseIds) + { + // Names, not entries: FileNames reads the buffer keys without pulling the deferred + // milestone, which is the whole point of asking it rather than Value here. + var fileNames = rescored.FileNames; + var runNames = new string[fileNames.Count]; + for (int i = 0; i < runNames.Length; i++) + runNames[i] = fileNames[i]; + + // The classification is derived ONCE and then carried by the accumulator, because + // rebuilding it runs for minutes at 6.3M library entries. Same source and one-time + // logging as every other path. + ModelDiagnosticsReport.BuildClassificationFromLibrary(config, libraryById, ctx.LogInfo, + out var classByBaseId, out var pairByBaseId, out var entrapmentRatio); + var accumulator = new ModelDiagnosticsData.Accumulator(runNames, classByBaseId, + pairByBaseId, entrapmentRatio, config.RunFdr, config.FdrLevel, 2); + + // Pass A: the accumulator's fold and co-assignment's cutoff phase, sharing one read. + // Both are per-row reductions over the same rows, so the second phase of the panel is + // the only thing that has to wait for a second pass. + var coAssign = new ModelDiagnosticsData.CoAssignmentPassBuilder(runNames, 2, true, + stratumBaseIds); + int fileIdx = 0; + foreach (var kvp in rescored.StreamFiles(@"Folding pass-2 diagnostics")) + { + foreach (var e in kvp.Value) + { + accumulator.Add(fileIdx, e.ModifiedSequence, e.Charge, e.EntryId, e.IsDecoy, + e.Score, new FdrQValues(e.RunPrecursorQvalue, e.RunPeptideQvalue, + e.ExperimentPrecursorQvalue, e.ExperimentPeptideQvalue, e.Pep)); + } + ModelDiagnosticsData.ObserveCoAssignmentRun(coAssign, fileIdx, kvp.Value, + classByBaseId, config.RunFdr, config.FdrLevel); + fileIdx++; + } + + // Pass B: the panel's detection phase, which needs the boundary pass A folded. + var coAssignment = ModelDiagnosticsData.BuildCoAssignmentDetection( + coAssign, runNames, rescored.StreamFiles(@"Building pass-2 co-assignment"), + classByBaseId, ModelDiagnosticsReport.BuildPrecursorMzLookup(libraryById), + config.RunFdr, config.FdrLevel); + + ModelDiagnosticsReport.WritePass2AndFinalizeFromAccumulator( + accumulator, coAssignment, pass2Contributions, config, ctx.LogInfo, ValidityKey(ctx)); + } + /// /// Re-clamp experiment q to each precursor's best run q, as a FOLD then an APPLY. /// diff --git a/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs b/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs index 9531bb1371..3698a0d94f 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs @@ -62,6 +62,9 @@ public void TestModelDiagnosticsData() TestPassingSetHonorsFdrLevel(); TestCalibrationBuildCalFile(); TestStreamingAccumulatorMatchesBatch(); + TestStreamingAccumulatorMatchesBatchPass2(); + TestAccumulatorRefusesTheWrongPass(); + TestCoAssignmentRefusesAMisorderedStream(); TestPeakCoAssignment(); TestCompletenessStatesTheRightReason(); } @@ -779,6 +782,190 @@ private static void TestStreamingAccumulatorMatchesBatch() Assert.IsFalse(batchNM.HasEntrapment, @"no manifest -> is_decoy-only split, no entrapment"); } + // The pass-2 half of the same claim, and the one that lets SecondPassFDR stream: the + // accumulator's BuildPass2 over folded reductions must byte-match the batch BuildPass2 + // over the resident survivor pool. This pins BOTH halves of the streamed report in one + // comparison, because the batch path builds its co-assignment panel with the ONE-CALL + // BuildCoAssignment while the streamed side drives the same panel through the split + // ObserveCoAssignmentRun / BuildCoAssignmentDetection phases the join uses - so a + // divergence in either the fold or the phase split reds this assert. + // + // The fixture carries apex RTs and a precursor m/z lookup as well as entrapment, because + // a null CoAssignment would let the panel half of the comparison pass vacuously. + private static void TestStreamingAccumulatorMatchesBatchPass2() + { + var cls = new Dictionary(); + var pair = new Dictionary(); + var mz = new Dictionary(); + var f1 = new List(); + var f2 = new List(); + for (int i = 0; i < 6; i++) + { + uint tid = (uint)(100 + i); + // Distinct apex RTs, and pairs of targets close enough in m/z to co-assign, so + // the panel has real shared peaks rather than an empty verdict. + int mzGroup = i / 2; // deliberate integer division: consecutive pairs share an m/z + mz[tid] = 500.0 + 0.004 * mzGroup; + mz[tid | DECOY_BIT] = 500.0 + 0.004 * mzGroup; + f1.Add(CoEntry(tid, false, 8.0 - i, 0.001 * (i + 1), "T" + i, 2, 10.0 + 0.01 * i, 8.0 - i)); + f1.Add(CoEntry(tid | DECOY_BIT, true, 1.0 + 0.1 * i, 0.5, "D" + i, 2, 12.0 + 0.01 * i, 1.0 + 0.1 * i)); + f2.Add(CoEntry(tid, false, 7.5 - i, 0.002 * (i + 1), "T" + i, 2, 30.0 + 0.01 * i, 8.0 - i)); + cls[tid] = EntrapmentClass.Target; + pair[tid] = (uint)i; + } + for (int i = 0; i < 4; i++) + { + uint pid = (uint)(200 + i); + int mzGroup = i / 2; // as above: the entrapment rows share m/z with a target pair + mz[pid] = 500.0 + 0.004 * mzGroup; + f1.Add(CoEntry(pid, false, 5.5 - i, 0.003 + 0.001 * i, "P" + i, 2, 10.002 + 0.01 * i, 5.5 - i)); + cls[pid] = EntrapmentClass.PTarget; + pair[pid] = (uint)i; + } + + var perFileEntries = WrapFiles(f1, f2); + const double r = 1.0, runFdr = 0.01; + const FdrLevel level = FdrLevel.Precursor; + System.Func mzLookup = id => mz.TryGetValue(id, out double v) ? v : double.NaN; + + var infos = new[] + { + new OspreyFeatureInfo("f0", "Feature Zero", false), + new OspreyFeatureInfo("f1", "Feature One", false), + }; + var facc = new FeatureContributions.Accumulator(2, true); + for (int i = 0; i < 10; i++) facc.Add(new[] { 2.0, 0.5 }, false); + for (int i = 0; i < 10; i++) facc.Add(new[] { -1.0, 0.0 }, true); + var contrib = facc.Build(new List { new[] { 2.0, -1.0 } }, infos); + + // Batch build (the resident-path oracle). + var batch = ModelDiagnosticsData.BuildPass2(perFileEntries, contrib, cls, pair, r, + runFdr, level, mzLookup); + + // Streamed build: one pass folding the accumulator AND the panel's cutoff phase + // together (what the join does), then a second pass for the panel's detection phase. + var runNames = perFileEntries.Select(kv => kv.Key).ToArray(); + var acc = new ModelDiagnosticsData.Accumulator(runNames, cls, pair, r, runFdr, level, 2); + var coAssign = new ModelDiagnosticsData.CoAssignmentPassBuilder(runNames, 2, true); + for (int fi = 0; fi < perFileEntries.Count; fi++) + { + foreach (var e in perFileEntries[fi].Value) + { + acc.Add(fi, e.ModifiedSequence, e.Charge, e.EntryId, e.IsDecoy, e.Score, + new FdrQValues(e.RunPrecursorQvalue, e.RunPeptideQvalue, + e.ExperimentPrecursorQvalue, e.ExperimentPeptideQvalue, 0.0)); + } + ModelDiagnosticsData.ObserveCoAssignmentRun(coAssign, fi, + perFileEntries[fi].Value, cls, runFdr, level); + } + var panel = ModelDiagnosticsData.BuildCoAssignmentDetection(coAssign, runNames, + perFileEntries, cls, mzLookup, runFdr, level); + var streamed = acc.BuildPass2(contrib, panel); + + var settings = new JsonSerializerSettings + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + FloatFormatHandling = FloatFormatHandling.Symbol, + FloatParseHandling = FloatParseHandling.Double, + }; + Assert.AreEqual( + JsonConvert.SerializeObject(batch, settings), + JsonConvert.SerializeObject(streamed, settings), + @"streamed pass-2 accumulator must byte-match the resident batch BuildPass2"); + + // Guard against a vacuous all-null match: every card the comparison covers must + // actually be populated by this fixture. + Assert.IsNotNull(batch.CoAssignment); + Assert.IsNotNull(batch.FdpViews); + Assert.IsTrue(batch.FdpViews.Count > 0); + Assert.IsNotNull(batch.CrossRun); + Assert.IsNotNull(batch.IdYield); + Assert.IsNotNull(batch.PerFile); + Assert.IsNotNull(batch.Model); + Assert.IsNotNull(batch.DensityRatio); + Assert.IsNotNull(batch.WinFraction); + + // Transfer mode: no retrained model, so the structural half stays null on BOTH arms. + // Worth pinning separately because it is the arm protein-compact actually runs, and + // a streamed build that invented a Model there would go unnoticed by the assert above. + var batchT = ModelDiagnosticsData.BuildPass2(perFileEntries, null, cls, pair, r, + runFdr, level, mzLookup); + var accT = new ModelDiagnosticsData.Accumulator(runNames, cls, pair, r, runFdr, level, 2); + var coAssignT = new ModelDiagnosticsData.CoAssignmentPassBuilder(runNames, 2, true); + for (int fi = 0; fi < perFileEntries.Count; fi++) + { + foreach (var e in perFileEntries[fi].Value) + { + accT.Add(fi, e.ModifiedSequence, e.Charge, e.EntryId, e.IsDecoy, e.Score, + new FdrQValues(e.RunPrecursorQvalue, e.RunPeptideQvalue, + e.ExperimentPrecursorQvalue, e.ExperimentPeptideQvalue, 0.0)); + } + ModelDiagnosticsData.ObserveCoAssignmentRun(coAssignT, fi, + perFileEntries[fi].Value, cls, runFdr, level); + } + var panelT = ModelDiagnosticsData.BuildCoAssignmentDetection(coAssignT, runNames, + perFileEntries, cls, mzLookup, runFdr, level); + Assert.AreEqual( + JsonConvert.SerializeObject(batchT, settings), + JsonConvert.SerializeObject(accT.BuildPass2(null, panelT), settings), + @"streamed pass-2 accumulator must byte-match the batch build in transfer mode"); + Assert.IsNull(batchT.Model, @"transfer mode -> no retrained model, structural half null"); + Assert.IsNull(batchT.WinFraction); + } + + // The accumulator folds different state for each pass, so building it for the pass it was + // not constructed for would return a plausible object assembled from partly unfolded + // reductions. Both directions must refuse instead. + private static void TestAccumulatorRefusesTheWrongPass() + { + var runNames = new[] { @"f1" }; + var pass1 = new ModelDiagnosticsData.Accumulator(runNames, null, null, 1.0, 0.01, + FdrLevel.Precursor); + Assert.ThrowsException(() => pass1.BuildPass2(null, null)); + var pass2 = new ModelDiagnosticsData.Accumulator(runNames, null, null, 1.0, 0.01, + FdrLevel.Precursor, 2); + Assert.ThrowsException(() => pass2.Build(null)); + } + + // The co-assignment phases index the acceptance boundary BY RUN POSITION, so a second + // pass that yielded runs in a different order - or stopped short - would judge every row + // against another run's boundary and still produce a complete, plausible panel. Nothing + // downstream can detect that, so both guards must throw rather than degrade. + private static void TestCoAssignmentRefusesAMisorderedStream() + { + var mz = new Dictionary { { 1, 500.000 }, { 2, 500.004 } }; + var f1 = new List { CoEntry(1, false, 9.0, 0.001, "A", 2, 10.000) }; + var f2 = new List { CoEntry(2, false, 8.0, 0.001, "B", 2, 20.000) }; + var inOrder = WrapFiles(f1, f2); + var runNames = inOrder.Select(kv => kv.Key).ToArray(); + System.Func mzLookup = id => mz.TryGetValue(id, out double v) ? v : double.NaN; + + var builder = new ModelDiagnosticsData.CoAssignmentPassBuilder(runNames, 2, true); + for (int fi = 0; fi < inOrder.Count; fi++) + { + ModelDiagnosticsData.ObserveCoAssignmentRun(builder, fi, inOrder[fi].Value, + null, 0.01, FdrLevel.Precursor); + } + + // Same runs, swapped order on the detection pass. + var swapped = new List>> { inOrder[1], inOrder[0] }; + Assert.ThrowsException(() => + ModelDiagnosticsData.BuildCoAssignmentDetection(builder, runNames, swapped, null, + mzLookup, 0.01, FdrLevel.Precursor)); + + // And a short read, which would otherwise report the missing run as having found nothing. + var truncated = new List>> { inOrder[0] }; + var builder2 = new ModelDiagnosticsData.CoAssignmentPassBuilder(runNames, 2, true); + for (int fi = 0; fi < inOrder.Count; fi++) + { + ModelDiagnosticsData.ObserveCoAssignmentRun(builder2, fi, inOrder[fi].Value, + null, 0.01, FdrLevel.Precursor); + } + Assert.ThrowsException(() => + ModelDiagnosticsData.BuildCoAssignmentDetection(builder2, runNames, truncated, null, + mzLookup, 0.01, FdrLevel.Precursor)); + } + /// /// BuildCalFile shapes one file's raw calibration ingredients into a CalFileRow: /// the LDA contribution table (weighted share, sorted, reds a negative row), the diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index b269902cb3..94170c2923 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -1051,6 +1051,15 @@ the text says so rather than describing the current shape as though it were the `.scores.parquet` and first-pass sidecar. The streamed path is the default; the switch goes when the resident one does. + `OSPREY_STAGE7_STREAM=0` is the Stage 7 sibling, and the same disposition applies. It + selects the resident second-pass join, where `RescoredEntries` holds every run's + survivors instead of rebuilding one run at a time through `StreamFiles`. Both arms are + required to produce identical bytes, which is what makes the switch an A/B ORACLE rather + than a fallback: it is the only way to compare the two, because nothing in the output + distinguishes them. That is also why `ScoringTaskShared.CanStreamStage7Join` is the one + place the choice is made, and why mode 3 asserts the marker line naming the shape that + actually ran rather than inferring it from the output. + 5. **Whether the 500-run / 64 GB target is met.** It is not yet, and which stage binds is itself moving as each is fixed. The two TODOs above carry the current measurements; deliberately not restated here, because a number in a contract document is stale the diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 5f007dc9fa..1e337ea340 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -294,18 +294,22 @@ $knownResidentGaps = @( # process environment, so a developer running the gate in their interactive shell would # otherwise silently lose an exported token for the rest of the session. $script:priorAllowResident = $env:OSPREY_ALLOW_UNFIXED_RESIDENT -# An operator running a deliberate A/B needs their token: OSPREY_STAGE6_STREAM_SURVIVORS=0 -# and OSPREY_FDR_PROJECTION=0 force resident paths ON PURPOSE, and clearing the token that -# admits them would abort the gate on its first leg with a guard error - making the very -# comparison this harness exists to support impossible to run. Ambient tokens are stripped -# ONLY when no such switch is set, which is the case the clearing is aimed at. -$abSwitchSet = ($env:OSPREY_STAGE6_STREAM_SURVIVORS -eq '0') -or ($env:OSPREY_FDR_PROJECTION -eq '0') +# An operator running a deliberate A/B needs their token: OSPREY_STAGE6_STREAM_SURVIVORS=0, +# OSPREY_FDR_PROJECTION=0 and OSPREY_STAGE7_STREAM=0 force resident paths ON PURPOSE, and +# clearing the token that admits them would abort the gate on its first leg with a guard +# error - making the very comparison this harness exists to support impossible to run. +# Ambient tokens are stripped ONLY when no such switch is set, which is the case the +# clearing is aimed at. +$abSwitchSet = ($env:OSPREY_STAGE6_STREAM_SURVIVORS -eq '0') -or + ($env:OSPREY_FDR_PROJECTION -eq '0') -or + ($env:OSPREY_STAGE7_STREAM -eq '0') if (-not [string]::IsNullOrWhiteSpace($env:OSPREY_ALLOW_UNFIXED_RESIDENT)) { if ($abSwitchSet) { # Extra parens: -f binds TIGHTER than +, so without them only the LAST fragment is # formatted and '{0}' survives verbatim into the output. Write-Host (("Keeping inherited OSPREY_ALLOW_UNFIXED_RESIDENT='{0}' - an A/B switch " + - "(OSPREY_STAGE6_STREAM_SURVIVORS/OSPREY_FDR_PROJECTION=0) is set and needs it.") ` + "(OSPREY_STAGE6_STREAM_SURVIVORS/OSPREY_FDR_PROJECTION/OSPREY_STAGE7_STREAM=0) " + + "is set and needs it.") ` -f $env:OSPREY_ALLOW_UNFIXED_RESIDENT) -ForegroundColor Yellow } else { Write-Host (("Clearing inherited OSPREY_ALLOW_UNFIXED_RESIDENT='{0}' - no leg of this " + @@ -638,10 +642,12 @@ function Get-DatasetCliArgs { if ($null -eq $Spec) { return $extra } if ($Spec.DecoysInLibrary) { $extra += '--decoys-in-library' } if ($Manifest) { $extra += @('--decoy-pairing-manifest', $Manifest) } - # --model-diagnostics is verified output-neutral (it routes the 2nd pass down - # the resident path instead of the FDR projection, and the two agree - # byte-for-byte), so it can ride on the golden-compared run rather than - # needing a second invocation. It populates the Pass 1 AND Pass 2 FDP views on + # --model-diagnostics is verified output-neutral, so it can ride on the + # golden-compared run rather than needing a second invocation. It no longer + # forces the 2nd pass down the resident path either: the pass-2 report is + # folded run by run through ModelDiagnosticsData.Accumulator, so a run with + # this flag streams the Stage 7 join exactly as one without it does, and + # mode 3 asserts that on every dataset. It populates the Pass 1 AND Pass 2 FDP views on # its own: --fdrbench-pass selects which pass an FDRBench INPUT FILE is written # for and does nothing at all without --fdrbench (OspreyCommandArgs warns, and # FdrBenchInputWriter returns early on an empty output path), so passing it here @@ -1922,6 +1928,20 @@ foreach ($name in $selected) { } } + # The pass-2 diagnostics product, chain vs straight-through - which on this suite is + # STREAMED vs RESIDENT, and is the only leg that compares the two shapes of the pass-2 + # report against each other. Straight-through has no --input-scores, so it takes the + # resident join; the chain's phase 4 is --task SecondPassFDR and takes the streamed one. + # The claim the fold rests on is that those produce the same report, and until this leg + # existed nothing checked it: mode 3 compares the blib and the FDR sidecars, and every + # diagnostics leg (1b, 5, 7) compares only the RESIDENT arm against a golden. + # + # A byte compare is right here: Pass2Data carries no timestamp and no version - those + # live on the pass-1 object - so the file is a pure function of the reported pool. If a + # field that legitimately varies per run is ever added to it, this leg fails LOUDLY and + # names the offset, which is the outcome to want; do not soften it to a field compare. + # Absence on either side is a FAILURE, not a skip: a route that stopped writing the + # product would otherwise pass by having nothing to compare. # Liveness: a comparison that verified nothing is not a passing comparison. Empty or # absent sidecars satisfy every field check trivially while breaking every resume, # and the rest of this harness fails closed on the same shape (Invoke-ResumeInvalidation @@ -1981,16 +2001,24 @@ foreach ($name in $selected) { # the bytes to reveal it. Without this, a change that silently disqualifies the streamed # arm leaves every leg green while the O(runs x entries) peak comes back. # - # Scoped to the datasets that can actually stream: CanStreamStage7Join declines under - # --model-diagnostics, which this suite sets on every dataset but plain Stellar, so - # demanding the line elsewhere would fail runs for a contract they cannot make. That - # narrowness is itself the finding - three of four datasets exercise only the resident - # arm - and it is recorded in the TODO rather than papered over here. - $chainCanStream = -not $cfg.ModelDiagnostics + # Demanded on EVERY dataset now. This used to be scoped to `-not $cfg.ModelDiagnostics` + # because CanStreamStage7Join declined under --model-diagnostics, which this suite sets + # on every dataset but plain Stellar - so three of the four exercised only the resident + # arm and a streamed-arm defect needing library decoys, entrapment or hram data passed + # the suite green. That term is gone from the predicate (the pass-2 report is folded run + # by run through ModelDiagnosticsData.Accumulator), so the scoping goes with it. + # + # Nothing else in the predicate varies across these datasets: --fdrbench is never passed + # here, so NeedsResidentPool is false, and protein-compact is the default pass-2 mode, so + # every mode-3 chain meets the remaining conditions structurally. A dataset that ever + # needs a narrower rule should name the condition rather than restore a blanket skip. + # ...unless the operator deliberately forced the resident arm for an A/B, in which case + # the run is doing exactly what it was asked to and demanding the marker would fail it + # for complying. This is the same allowance $abSwitchSet makes for the resident tokens. $chainStreamed = Select-String -Path (Join-Path (Join-Path $chainRoot 'logs') 'phase4.log') ` -Pattern 'Second-pass join: folding over \d+ run\(s\)' -Quiet - if (-not $chainCanStream) { - $summaryLines.Add("$name mode3 (streamed join): SKIP (--model-diagnostics keeps the resident pool)") + if ($env:OSPREY_STAGE7_STREAM -eq '0') { + $summaryLines.Add("$name mode3 (streamed join): SKIP (OSPREY_STAGE7_STREAM=0 forces the resident arm)") } elseif (-not $chainStreamed) { $overallFail = $true Write-Problem-Tc ("$name mode3 (streamed join): FAIL - phase 4 did not report the " + From 2a8c198c0a23630e495fe0dc3c675eb3cb181b08 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 17:39:15 -0700 Subject: [PATCH 14/30] Moved the pass-1 product check ahead of the pass-2 diagnostics fold * Checked for the pass-1 sidecar before folding rather than leaving it to the report writer, which is called after both stream passes have already run * Without this a bed with no pass-1 product rebuilt all 446 runs twice and then discarded the result; the resident path already checked first See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 5fa007a0f1..85d0cfdff2 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -572,6 +572,20 @@ private void WritePass2DiagnosticsStreamedCore(PipelineContext ctx, RescoredEntr IReadOnlyDictionary libraryById, OspreyConfig config, HashSet stratumBaseIds) { + // The pass-1 product is what pass 2 ENRICHES, so its absence means there is nothing + // for this fold to attach to - and the fold is two full passes over the stream, + // rebuilding every run from disk twice. Checked HERE rather than left to the report + // writer, which is where the resident path checks it: there the read is the first + // statement and costs nothing, but on this path the writer is called AFTER the + // folding, so leaving the check to it spends both passes and discards the result. + // Measured shape at 446 runs: two rebuilds of 446 runs for an artifact that is then + // not written. Same message the writer emits, so the log reads identically either way. + if (!File.Exists(ModelDiagnosticsReport.Pass1SidecarPath(config))) + { + ctx.LogInfo(@"[MODEL-DIAGNOSTICS] pass-1 data sidecar not found; pass-2 enrichment skipped (pass-1 page stands)."); + return; + } + // Names, not entries: FileNames reads the buffer keys without pulling the deferred // milestone, which is the whole point of asking it rather than Value here. var fileNames = rescored.FileNames; From 89069ca270aacdc54df82022015a7b390e4dcba6 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 19:06:30 -0700 Subject: [PATCH 15/30] Addressed code review findings on the pass-2 diagnostics fold * Removed an orphaned comment left behind when the chain-vs-straight diagnostics comparison was dropped, and recorded why it was dropped * Reverted OSPREY_STAGE7_STREAM from the gate's A/B token set: it arms no resident-pool guard, so admitting it weakened the named-token ratchet * Scoped the streamed-join assertion to CanStreamStage7Join's own terms, so a projection-off or transfer run is no longer failed for complying * Narrowed the diagnostics try/catch to the co-assignment panel, so a refused panel no longer discards the eight correctly folded cards * Verified run order in the fold pass, which indexes the per-file counts and cross-run streams and not only the panel's boundary * Reused BuildModelDiagnosticsAccumulator and FeedModelDiagnostics instead of re-implementing them, and read the pass-1 product once * Guarded the co-assignment builder against post-seal mutation and a second seal, both newly reachable once the phases became public * Covered the protein-compact stratum in the byte-identity test, the only configuration the streamed arm runs in production * Corrected the doc claim that OSPREY_STAGE7_STREAM is an in-place A/B, and a stale resume error blaming --model-diagnostics See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../ModelDiagnosticsData.CoAssignment.cs | 46 ++++++++++ .../Osprey/Osprey.Tasks/FirstPassFdrTask.cs | 12 ++- .../ModelDiagnosticsReport.cs | 9 +- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 14 +-- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 87 ++++++++++++------- .../Osprey.Test/ModelDiagnosticsDataTest.cs | 41 +++++++++ .../Osprey/docs/00-pipeline-architecture.md | 20 +++-- pwiz_tools/Osprey/regression.ps1 | 73 ++++++++++------ 8 files changed, 230 insertions(+), 72 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs index cebf237001..12de015dfe 100644 --- a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs +++ b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.CoAssignment.cs @@ -572,6 +572,23 @@ public static CoAssignmentData BuildCoAssignmentDetection( /// another run's boundary and still produce a complete, plausible panel. Nothing /// downstream could detect it, which is why this throws rather than logs. /// + /// + /// for a caller driving the phases itself. Public because + /// the streamed second-pass join indexes MORE than the panel by that position - the + /// diagnostics accumulator's per-file counts and cross-run streams share it - so the + /// same assertion has to be available outside this file. + /// + public static void VerifyStreamedRun(string[] runNames, int index, string runName) + { + VerifyRunOrder(runNames, index, runName); + } + + /// for the same caller, for the same reason. + public static void VerifyStreamedRunCount(string[] runNames, int seen) + { + VerifyRunCount(runNames, seen); + } + private static void VerifyRunOrder(string[] runNames, int index, string runName) { if (index < runNames.Length && Equals(runNames[index], runName)) @@ -719,6 +736,16 @@ public void ObserveCutoff(int fileIdx, EntrapmentClass cls, uint entryId, double double experimentAggregateScore, double runQvalue, double experimentQvalue, double runFdr) { + // The forward misuse (judging before sealing) has always thrown; this is the + // REVERSE, which the public phase split newly admits. Observing after the seal + // mutates _experimentBest and lets SealRunCutoff overwrite a run's cutoff, moving + // the boundary that already-emitted verdicts were compared against - so the panel + // would mix two boundaries and still look complete. + if (_sealed) + { + throw new InvalidOperationException( + @"CoAssignmentPassBuilder.ObserveCutoff was called after SealCutoffs. The acceptance boundary is fixed once sealed, and moving it would leave verdicts already emitted against the old one."); + } if (fileIdx != _fileIdx) { if (_fileIdx >= 0) @@ -800,6 +827,14 @@ public void ObserveCutoff(int fileIdx, EntrapmentClass cls, uint entryId, double /// public void SealRunCutoff(int fileIdx) { + // Same reason as ObserveCutoff: this OVERWRITES _runCutoff[fileIdx] and + // _admittedRunDecoys[fileIdx] unconditionally, so after the seal it would move a + // per-run boundary out from under verdicts already compared against it. + if (_sealed) + { + throw new InvalidOperationException( + @"CoAssignmentPassBuilder.SealRunCutoff was called after SealCutoffs. A run's boundary cannot move once the detection phase has begun judging rows against it."); + } double min = double.NaN; foreach (uint id in _fileAccepted) { @@ -844,6 +879,17 @@ public void SealRunCutoff(int fileIdx) /// public void SealCutoffs() { + // Sealing twice is not idempotent and fails SILENTLY: the tail of this method + // records _acceptedForCutoff from _experimentAccepted and then CLEARS it, so a + // second call sets that count to 0 while _experimentCutoff keeps the boundary it + // already drew - and every row is then judged, added and flushed a second time, + // doubling the panel. Unreachable while both phases lived behind one private + // builder; reachable the moment they became a two-call public sequence. + if (_sealed) + { + throw new InvalidOperationException( + @"CoAssignmentPassBuilder.SealCutoffs was called twice. The detection phase seals the boundary itself, so driving it a second time would re-count the whole pool against a boundary already drawn."); + } foreach (uint id in _experimentAccepted) { if (!_experimentBest.TryGetValue(id, out double v)) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs index 324e368173..ce696831e2 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs @@ -1622,12 +1622,19 @@ private static ModelDiagnosticsData.CalibrationData BuildCalibrationData( /// walks, which is what keeps the streamed report identical to the resident one. /// is passed rather than pulled from the context because /// the rehydrate caller runs before LibraryById is published. + /// + /// pass selects which reductions the accumulator folds: 1 for the + /// pre-compaction first pass, 2 for SecondPassFdrTask's streamed fold over the final + /// reported pool. Everything else - the classification, the run-name seeding, the run FDR + /// and level - is derived identically for both, which is why the second pass shares this + /// helper rather than re-deriving them and letting the two drift. /// internal static ModelDiagnosticsData.Accumulator BuildModelDiagnosticsAccumulator( IReadOnlyList fileNames, IReadOnlyDictionary libraryById, OspreyConfig config, - Action logInfo) + Action logInfo, + int pass = 1) { ModelDiagnosticsReport.BuildClassificationFromLibrary(config, libraryById, logInfo, out var classByBaseId, out var pairByBaseId, out var entrapmentRatio); @@ -1635,7 +1642,8 @@ internal static ModelDiagnosticsData.Accumulator BuildModelDiagnosticsAccumulato for (int i = 0; i < runNames.Length; i++) runNames[i] = fileNames[i]; return new ModelDiagnosticsData.Accumulator( - runNames, classByBaseId, pairByBaseId, entrapmentRatio, config.RunFdr, config.FdrLevel); + runNames, classByBaseId, pairByBaseId, entrapmentRatio, config.RunFdr, + config.FdrLevel, pass); } /// diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs b/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs index c6d9bf9d9b..5eb6321288 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs @@ -283,8 +283,14 @@ public static void WritePass2AndFinalize( /// that its per-row verdicts are then compared against - it cannot be folded in the pass /// that computes it. Null leaves the panel out, exactly as a null library lookup does on /// the resident path. + /// + /// data is the pass-1 object the caller already read in order to decide + /// whether folding was worth doing at all. It is handed in rather than re-read here: + /// on this path that read is the PRECONDITION for two full stream passes, so it has to + /// happen before them, and reading it a second time would let the two reads disagree. /// public static void WritePass2AndFinalizeFromAccumulator( + ModelDiagnosticsData data, ModelDiagnosticsData.Accumulator accumulator, ModelDiagnosticsData.CoAssignmentData coAssignment, FeatureContributions pass2Contributions, @@ -294,7 +300,6 @@ public static void WritePass2AndFinalizeFromAccumulator( { try { - var data = ReadPass1ForEnrichment(config, logInfo); if (data == null) return; data.Pass2 = accumulator.BuildPass2(pass2Contributions, coAssignment); @@ -311,7 +316,7 @@ public static void WritePass2AndFinalizeFromAccumulator( /// explaining that the pass-1 page stands unchanged. Absence is a degrade, not a failure: /// pass 1's page is a complete statement of the first pass on its own. /// - private static ModelDiagnosticsData ReadPass1ForEnrichment(OspreyConfig config, + public static ModelDiagnosticsData ReadPass1ForEnrichment(OspreyConfig config, Action logInfo) { var data = ReadJson(ResolvePass1SidecarPath(config)); diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index d8a2b479d7..2999ba8ef5 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -447,15 +447,19 @@ public override bool Run(PipelineContext ctx) bool noRescorePossible = rescoreBundle == null && !perRunPlanAvailable; if (!didPlan && noRescorePossible && pass2Present < pass2Expected) { + // No --model-diagnostics clause. It used to append "because --model-diagnostics + // keeps the all-runs hydrate", which was true when that flag excluded the per-run + // hydrate and is now false on both counts: CanHydratePerRun stopped excluding it, + // and the Stage 7 join no longer declines under it either. An operator told that + // would drop the flag, re-run for hours and hit the identical refusal, because + // the actual cause is the one the sentence already names - no plan, no bundle, + // no per-run source. ctx.LogError(string.Format( @"Rescore resume: {0} of {1} run(s) still need re-scoring, but this process has " + @"no plan to do it - FirstPassFDR did not plan here, no worker bundle was " + - @"supplied, and the per-run hydrate is unavailable{2}. Continuing would write " + + @"supplied, and the per-run hydrate is unavailable. Continuing would write " + @"an output silently missing those runs.", - pass2Expected - pass2Present, pass2Expected, - ctx.Config.ModelDiagnostics - ? @" because --model-diagnostics keeps the all-runs hydrate" - : string.Empty)); + pass2Expected - pass2Present, pass2Expected)); ctx.ExitCode = 1; return false; } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 85d0cfdff2..d519218524 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -574,17 +574,19 @@ private void WritePass2DiagnosticsStreamedCore(PipelineContext ctx, RescoredEntr { // The pass-1 product is what pass 2 ENRICHES, so its absence means there is nothing // for this fold to attach to - and the fold is two full passes over the stream, - // rebuilding every run from disk twice. Checked HERE rather than left to the report - // writer, which is where the resident path checks it: there the read is the first - // statement and costs nothing, but on this path the writer is called AFTER the - // folding, so leaving the check to it spends both passes and discards the result. - // Measured shape at 446 runs: two rebuilds of 446 runs for an artifact that is then - // not written. Same message the writer emits, so the log reads identically either way. - if (!File.Exists(ModelDiagnosticsReport.Pass1SidecarPath(config))) - { - ctx.LogInfo(@"[MODEL-DIAGNOSTICS] pass-1 data sidecar not found; pass-2 enrichment skipped (pass-1 page stands)."); + // rebuilding every run from disk twice. Read HERE rather than left to the report + // writer, which is where the resident path reads it: there the read is the first + // statement and costs nothing, but on this path the writer runs AFTER the folding, + // so leaving it there spends both passes and discards the result - two rebuilds of + // 446 runs for an artifact that is then not written. + // + // The READ, not File.Exists. The condition that actually matters is "can this be + // deserialized into something to enrich": an empty file deserializes to null and a + // truncated one throws, and both are exactly what an interrupted run leaves behind. + // A presence check would pass on either and spend everything anyway. + var pass1Data = ModelDiagnosticsReport.ReadPass1ForEnrichment(config, ctx.LogInfo); + if (pass1Data == null) return; - } // Names, not entries: FileNames reads the buffer keys without pulling the deferred // milestone, which is the whole point of asking it rather than Value here. @@ -593,13 +595,15 @@ private void WritePass2DiagnosticsStreamedCore(PipelineContext ctx, RescoredEntr for (int i = 0; i < runNames.Length; i++) runNames[i] = fileNames[i]; - // The classification is derived ONCE and then carried by the accumulator, because - // rebuilding it runs for minutes at 6.3M library entries. Same source and one-time - // logging as every other path. - ModelDiagnosticsReport.BuildClassificationFromLibrary(config, libraryById, ctx.LogInfo, - out var classByBaseId, out var pairByBaseId, out var entrapmentRatio); - var accumulator = new ModelDiagnosticsData.Accumulator(runNames, classByBaseId, - pairByBaseId, entrapmentRatio, config.RunFdr, config.FdrLevel, 2); + // The SAME seeding the first pass uses, differing only in the pass argument. Built + // through the shared helper rather than re-derived here: the classification runs for + // minutes at 6.3M library entries, and a second copy of the FdrEntry-to-accumulator + // mapping is exactly how the two passes drift apart without anything noticing. + // ClassByBaseId is read back off the accumulator for the same reason - the panel and + // the fold must classify a row identically or they describe different pools. + var accumulator = FirstPassFdrTask.BuildModelDiagnosticsAccumulator( + fileNames, libraryById, config, ctx.LogInfo, 2); + var classByBaseId = accumulator.ClassByBaseId; // Pass A: the accumulator's fold and co-assignment's cutoff phase, sharing one read. // Both are per-row reductions over the same rows, so the second phase of the panel is @@ -609,25 +613,50 @@ private void WritePass2DiagnosticsStreamedCore(PipelineContext ctx, RescoredEntr int fileIdx = 0; foreach (var kvp in rescored.StreamFiles(@"Folding pass-2 diagnostics")) { - foreach (var e in kvp.Value) - { - accumulator.Add(fileIdx, e.ModifiedSequence, e.Charge, e.EntryId, e.IsDecoy, - e.Score, new FdrQValues(e.RunPrecursorQvalue, e.RunPeptideQvalue, - e.ExperimentPrecursorQvalue, e.ExperimentPeptideQvalue, e.Pep)); - } + // Same assertion phase 2 makes, and for a WIDER reason: this index addresses the + // accumulator's per-file passing counts and its four cross-run streams as well as + // the panel's boundary, and every one of those is reported against _runNames[f]. + // A stream that reordered or dropped a run would mis-attribute the PerFile and + // CrossRun cards and still produce a complete, plausible Pass2Data - and phase 2 + // could not see it, because it re-derives its own indices from the same array. + ModelDiagnosticsData.VerifyStreamedRun(runNames, fileIdx, kvp.Key); + ScoringTaskShared.FeedModelDiagnostics(accumulator, fileIdx, kvp.Value); ModelDiagnosticsData.ObserveCoAssignmentRun(coAssign, fileIdx, kvp.Value, classByBaseId, config.RunFdr, config.FdrLevel); fileIdx++; } + ModelDiagnosticsData.VerifyStreamedRunCount(runNames, fileIdx); // Pass B: the panel's detection phase, which needs the boundary pass A folded. - var coAssignment = ModelDiagnosticsData.BuildCoAssignmentDetection( - coAssign, runNames, rescored.StreamFiles(@"Building pass-2 co-assignment"), - classByBaseId, ModelDiagnosticsReport.BuildPrecursorMzLookup(libraryById), - config.RunFdr, config.FdrLevel); + // + // Caught HERE, around this call alone, and not by the method-wide guard. The + // co-assignment order/count checks throw BY DESIGN, and the intended outcome of one + // firing is that the PANEL is refused - not that the eight correctly folded cards go + // with it. Letting the throw reach the outer catch would unwind past the writer + // below and leave no 2nd-pass product at all, which is how the resident arm behaves + // when its panel cannot be built (it degrades to CoAssignment = null and still + // writes everything). The two arms have to fail the same way, or the byte-identity + // claim only holds on the happy path. + ModelDiagnosticsData.CoAssignmentData coAssignment = null; + try + { + coAssignment = ModelDiagnosticsData.BuildCoAssignmentDetection( + coAssign, runNames, rescored.StreamFiles(@"Building pass-2 co-assignment"), + classByBaseId, ModelDiagnosticsReport.BuildPrecursorMzLookup(libraryById), + config.RunFdr, config.FdrLevel); + } + catch (Exception ex) + { + // Named separately from the outer handler: "the panel was refused" and "the + // whole enrichment failed" are different outcomes and used to log the same line. + ctx.LogInfo(string.Format( + @"[MODEL-DIAGNOSTICS] peak co-assignment refused, so the panel is omitted; the rest of the pass-2 report is unaffected: {0}", + ex.Message)); + } ModelDiagnosticsReport.WritePass2AndFinalizeFromAccumulator( - accumulator, coAssignment, pass2Contributions, config, ctx.LogInfo, ValidityKey(ctx)); + pass1Data, accumulator, coAssignment, pass2Contributions, config, ctx.LogInfo, + ValidityKey(ctx)); } /// @@ -911,7 +940,7 @@ private void WriteBlibOutput( nFallback)); } - // Streamed, and the LAST walk of the pool in this phase: what comes back is a + // Streamed, and the last walk the BLIB path makes: what comes back is a // compact record per passing observation plus the best run per precursor, so // everything after this line works on ~14 M values instead of holding 137 M // entries alive to read eight fields off them (#4486). diff --git a/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs b/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs index 3698a0d94f..89cde0f3b7 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs @@ -911,6 +911,47 @@ private static void TestStreamingAccumulatorMatchesBatchPass2() @"streamed pass-2 accumulator must byte-match the batch build in transfer mode"); Assert.IsNull(batchT.Model, @"transfer mode -> no retrained model, structural half null"); Assert.IsNull(batchT.WinFraction); + + // STRATIFIED, which is the only shape the streamed arm ever runs in production: + // CanStreamStage7Join requires protein-compact, and under it SecondPassFdrTask always + // passes the ProteinCompactStratum to the panel. A non-null stratum SPLITS the + // experiment acceptance boundary in two (issue #4573), which is the most state- and + // position-sensitive part of the panel and the part the phase split newly drives - + // so comparing only the pooled-boundary path would leave the production + // configuration with no equivalence coverage at all. An EMPTY stratum is a real + // (degenerate) configuration and deliberately NOT the same as no stratum, so this + // uses a populated one. + var stratum = new HashSet(); + for (int i = 0; i < 4; i++) + stratum.Add((uint)(100 + i)); // half the targets in, half out + var batchS = ModelDiagnosticsData.BuildPass2(perFileEntries, contrib, cls, pair, r, + runFdr, level, mzLookup, stratum); + var accS = new ModelDiagnosticsData.Accumulator(runNames, cls, pair, r, runFdr, level, 2); + var coAssignS = new ModelDiagnosticsData.CoAssignmentPassBuilder(runNames, 2, true, stratum); + for (int fi = 0; fi < perFileEntries.Count; fi++) + { + foreach (var e in perFileEntries[fi].Value) + { + accS.Add(fi, e.ModifiedSequence, e.Charge, e.EntryId, e.IsDecoy, e.Score, + new FdrQValues(e.RunPrecursorQvalue, e.RunPeptideQvalue, + e.ExperimentPrecursorQvalue, e.ExperimentPeptideQvalue, 0.0)); + } + ModelDiagnosticsData.ObserveCoAssignmentRun(coAssignS, fi, + perFileEntries[fi].Value, cls, runFdr, level); + } + var panelS = ModelDiagnosticsData.BuildCoAssignmentDetection(coAssignS, runNames, + perFileEntries, cls, mzLookup, runFdr, level); + Assert.AreEqual( + JsonConvert.SerializeObject(batchS, settings), + JsonConvert.SerializeObject(accS.BuildPass2(contrib, panelS), settings), + @"streamed pass-2 accumulator must byte-match the batch build under a protein-compact stratum"); + // The stratum has to have actually split the boundary, or this arm passes vacuously + // by reproducing the pooled result twice. + Assert.IsNotNull(batchS.CoAssignment); + Assert.AreNotEqual( + JsonConvert.SerializeObject(batch.CoAssignment, settings), + JsonConvert.SerializeObject(batchS.CoAssignment, settings), + @"a populated stratum must move the panel, or this arm proves nothing"); } // The accumulator folds different state for each pass, so building it for the pass it was diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index 94170c2923..8697600959 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -1051,14 +1051,22 @@ the text says so rather than describing the current shape as though it were the `.scores.parquet` and first-pass sidecar. The streamed path is the default; the switch goes when the resident one does. - `OSPREY_STAGE7_STREAM=0` is the Stage 7 sibling, and the same disposition applies. It + `OSPREY_STAGE7_STREAM=0` is the Stage 7 sibling, and the same disposition applies - it selects the resident second-pass join, where `RescoredEntries` holds every run's survivors instead of rebuilding one run at a time through `StreamFiles`. Both arms are - required to produce identical bytes, which is what makes the switch an A/B ORACLE rather - than a fallback: it is the only way to compare the two, because nothing in the output - distinguishes them. That is also why `ScoringTaskShared.CanStreamStage7Join` is the one - place the choice is made, and why mode 3 asserts the marker line naming the shape that - actually ran rather than inferring it from the output. + required to produce identical bytes. + + **It is NOT the in-place A/B its Stage 6 sibling is, and must not be described as one.** + `CanStreamStage7Join` short-circuits on `!config.ExpectReconciledInput` *before* it reads + the switch, and that flag is set only for `--task SecondPassFDR`. So on a straight-through + run the switch changes nothing - while `SecondPassFdrTask.ValidityKey` appends + `;stage7stream=0` unconditionally, invalidating the `.blib` and every 2nd-pass sidecar and + forcing a full Stage 7 re-run for a setting that cannot change the arm. Comparing the two + shapes means comparing two `--task SecondPassFDR` runs over the same linked bed. + + Because nothing in the output distinguishes the arms, the shape that ran is asserted from + the marker line `Second-pass join: folding over N run(s)` rather than inferred - which is + what mode 3 does, scoped to the configurations that can actually stream. 5. **Whether the 500-run / 64 GB target is met.** It is not yet, and which stage binds is itself moving as each is fixed. The two TODOs above carry the current measurements; diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 1e337ea340..91c368ea66 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -294,22 +294,28 @@ $knownResidentGaps = @( # process environment, so a developer running the gate in their interactive shell would # otherwise silently lose an exported token for the rest of the session. $script:priorAllowResident = $env:OSPREY_ALLOW_UNFIXED_RESIDENT -# An operator running a deliberate A/B needs their token: OSPREY_STAGE6_STREAM_SURVIVORS=0, -# OSPREY_FDR_PROJECTION=0 and OSPREY_STAGE7_STREAM=0 force resident paths ON PURPOSE, and -# clearing the token that admits them would abort the gate on its first leg with a guard -# error - making the very comparison this harness exists to support impossible to run. -# Ambient tokens are stripped ONLY when no such switch is set, which is the case the -# clearing is aimed at. +# An operator running a deliberate A/B needs their token: OSPREY_STAGE6_STREAM_SURVIVORS=0 +# and OSPREY_FDR_PROJECTION=0 force resident paths ON PURPOSE, and clearing the token that +# admits them would abort the gate on its first leg with a guard error - making the very +# comparison this harness exists to support impossible to run. Ambient tokens are stripped +# ONLY when no such switch is set, which is the case the clearing is aimed at. +# +# OSPREY_STAGE7_STREAM=0 is deliberately NOT in this set, though it also forces a resident +# path. The list is not "switches that select a resident path", it is "switches that arm a +# guard which REFUSES without a token" - ResidentPaths.KNOWN_UNFIXED is exactly +# { FDRBENCH_PASS1, NON_PERCOLATOR_FDR, PROJECTION_OFF, COMPACTED_ENTRIES_BUFFER } and has +# no Stage-7 entry, which is also why $knownResidentGaps records the Stage-7 pool with +# Token = 'NONE'. Adding it would keep an ambient OSPREY_ALLOW_UNFIXED_RESIDENT alive across +# every leg of every dataset in exchange for nothing, which is the named-token ratchet the +# preamble above exists to enforce, weakened. $abSwitchSet = ($env:OSPREY_STAGE6_STREAM_SURVIVORS -eq '0') -or - ($env:OSPREY_FDR_PROJECTION -eq '0') -or - ($env:OSPREY_STAGE7_STREAM -eq '0') + ($env:OSPREY_FDR_PROJECTION -eq '0') if (-not [string]::IsNullOrWhiteSpace($env:OSPREY_ALLOW_UNFIXED_RESIDENT)) { if ($abSwitchSet) { # Extra parens: -f binds TIGHTER than +, so without them only the LAST fragment is # formatted and '{0}' survives verbatim into the output. Write-Host (("Keeping inherited OSPREY_ALLOW_UNFIXED_RESIDENT='{0}' - an A/B switch " + - "(OSPREY_STAGE6_STREAM_SURVIVORS/OSPREY_FDR_PROJECTION/OSPREY_STAGE7_STREAM=0) " + - "is set and needs it.") ` + "(OSPREY_STAGE6_STREAM_SURVIVORS/OSPREY_FDR_PROJECTION=0) is set and needs it.") ` -f $env:OSPREY_ALLOW_UNFIXED_RESIDENT) -ForegroundColor Yellow } else { Write-Host (("Clearing inherited OSPREY_ALLOW_UNFIXED_RESIDENT='{0}' - no leg of this " + @@ -1928,20 +1934,20 @@ foreach ($name in $selected) { } } - # The pass-2 diagnostics product, chain vs straight-through - which on this suite is - # STREAMED vs RESIDENT, and is the only leg that compares the two shapes of the pass-2 - # report against each other. Straight-through has no --input-scores, so it takes the - # resident join; the chain's phase 4 is --task SecondPassFDR and takes the streamed one. - # The claim the fold rests on is that those produce the same report, and until this leg - # existed nothing checked it: mode 3 compares the blib and the FDR sidecars, and every - # diagnostics leg (1b, 5, 7) compares only the RESIDENT arm against a golden. + # NOTHING compares the pass-2 diagnostics product across routes, and that is deliberate + # rather than an omission. A chain-vs-straight byte compare was tried here and removed: + # mode 3 contracts its sidecar comparison at 1e-9, not byte identity, so demanding the + # latter of an artifact DERIVED from those sidecars asserts more than the mode promises. + # It also went red on a real but unrelated defect - the two routes disagree on the paired + # entrapment FDP curve on the generated-decoy dataset (ProteoWizard/pwiz#4645), which + # reproduces with the streamed fold forced off and so is not the fold's doing. # - # A byte compare is right here: Pass2Data carries no timestamp and no version - those - # live on the pass-1 object - so the file is a pure function of the reported pool. If a - # field that legitimately varies per run is ever added to it, this leg fails LOUDLY and - # names the offset, which is the outcome to want; do not soften it to a field compare. - # Absence on either side is a FAILURE, not a skip: a route that stopped writing the - # product would otherwise pass by having nothing to compare. + # The streamed pass-2 report is covered instead by the marker assertion below (which + # shape ran) plus ModelDiagnosticsDataTest's byte-identity oracle over the accumulator. + # A gate-level A/B keyed on OSPREY_STAGE7_STREAM was considered and rejected: the intent + # is to REMOVE the ability not to stream, so a leg built on that switch would be built to + # be deleted. See #4645 for what to assert instead - the panel's INPUTS, not its curve. + # Liveness: a comparison that verified nothing is not a passing comparison. Empty or # absent sidecars satisfy every field check trivially while breaking every resume, # and the rest of this harness fails closed on the same shape (Invoke-ResumeInvalidation @@ -2012,13 +2018,24 @@ foreach ($name in $selected) { # here, so NeedsResidentPool is false, and protein-compact is the default pass-2 mode, so # every mode-3 chain meets the remaining conditions structurally. A dataset that ever # needs a narrower rule should name the condition rather than restore a blanket skip. - # ...unless the operator deliberately forced the resident arm for an A/B, in which case - # the run is doing exactly what it was asked to and demanding the marker would fail it - # for complying. This is the same allowance $abSwitchSet makes for the resident tokens. + # ...unless the run was asked for a configuration that cannot stream, in which case it + # is doing exactly what it was told and demanding the marker would fail it for + # complying. These are CanStreamStage7Join's OWN terms, not a mode list: the switch that + # forces the resident join, the two that make NeedsResidentPool true, and any pass-2 + # mode other than protein-compact (transfer still computes its per-file half in Stage 7). + # ExpectReconciledInput is not among them because phase 4 always sets it. + # Enumerated rather than inferred from the log, because a resident run says nothing + # about WHY it was resident - and a silent SKIP for the wrong reason is what this leg + # exists to prevent. + $chainCannotStream = + ($env:OSPREY_STAGE7_STREAM -eq '0') -or + ($env:OSPREY_FDR_PROJECTION -eq '0') -or + (-not [string]::IsNullOrWhiteSpace($env:OSPREY_PASS2_QVALUE) -and + $env:OSPREY_PASS2_QVALUE -ne 'protein-compact') $chainStreamed = Select-String -Path (Join-Path (Join-Path $chainRoot 'logs') 'phase4.log') ` -Pattern 'Second-pass join: folding over \d+ run\(s\)' -Quiet - if ($env:OSPREY_STAGE7_STREAM -eq '0') { - $summaryLines.Add("$name mode3 (streamed join): SKIP (OSPREY_STAGE7_STREAM=0 forces the resident arm)") + if ($chainCannotStream) { + $summaryLines.Add("$name mode3 (streamed join): SKIP (this configuration cannot stream the join)") } elseif (-not $chainStreamed) { $overallFail = $true Write-Problem-Tc ("$name mode3 (streamed join): FAIL - phase 4 did not report the " + From 20fb5c2744b1500a341484c51a219bd3a4424f4f Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 20:58:30 -0700 Subject: [PATCH 16/30] Fixed the two Copilot findings this branch actually introduces * Made a callback rejection mid-walk throw like a mid-read failure, so TryWalkRecords no longer returns false with records already overlaid * Cached the experiment sidecar load per generation, keyed on the file's identity, so the streamed pass-2 overlay stops deserializing it once per run on a resume that publishes no experiment scope Both arrived on this branch rather than from master, so merging would have shipped them. See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.IO/FdrScoresSidecar.cs | 9 ++- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 62 ++++++++++++++++++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs index 3f74f22a63..35c62406f1 100644 --- a/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.IO/FdrScoresSidecar.cs @@ -801,8 +801,15 @@ private static bool TryWalkRecords(string path, Pass expectedPass, remaining -= take; for (int rec = 0; rec < take; rec++) { + // Same answer as the mid-read failure above and the catch below, and + // for the same reason: this was the third exit from the walk and the + // only one that still returned false with records already applied. + // A callback that rejects record N has left N-1 records on the + // caller's entries, which is the half-pass-1/half-pass-2 state the + // remarks below describe - and TryRead rejects on a missing entry id, + // so it is reachable from a real sidecar, not just a hostile callback. if (!onRecord(chunk, rec * RecordLength)) - return false; + return delivered == 0 ? false : ThrowPartialWalk(path, delivered); delivered++; } } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index 8ad1b4ed4b..166032bac5 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -813,9 +813,65 @@ private static void RestorePass1Scalars( private static IReadOnlyDictionary LoadExperimentRecords( OspreyConfig config, FdrScoresSidecar.Pass pass) { - return LoadExperimentRecordsFrom( - FdrExperimentSidecar.PathFor(config?.OutputBlib, - ScoringTaskShared.ArtifactSiblingPath(config), pass), pass); + string path = FdrExperimentSidecar.PathFor(config?.OutputBlib, + ScoringTaskShared.ArtifactSiblingPath(config), pass); + return LoadExperimentRecordsCached(path, pass); + } + + /// + /// One deserialization per GENERATION of the experiment sidecar, rather than one per + /// call, keyed on the file's identity (path, length, last write) so a rewrite invalidates + /// the cache on its own. + /// + /// The caller that needs this is the streamed pass-2 overlay: it resolves the + /// records inside a per-run hook, deliberately (see + /// - Stage 7 crosses the + /// no-scope-to-scope boundary partway through, so capturing once would freeze + /// pre-competition values into every later fold). On a resume where the competition is + /// skipped and no Pass2ExperimentScope is ever published, that per-call resolution + /// falls through to disk for EVERY run of EVERY StreamFiles pass - and Stage 7 + /// makes many. At 446 runs that is O(runs x sidecar) deserialization of a file with 1.24 M + /// records, which is the shape this whole area exists to eliminate. + /// + /// Caching on identity rather than hoisting the call is what keeps the documented + /// semantics intact: protein FDR REWRITES this sidecar mid-Stage-7, and a rewrite changes + /// length or write time, so the next resolve re-reads. A plain hoist would have frozen the + /// pre-competition answer, which is the bug the per-call resolution was written to avoid. + /// One FileInfo stat per call replaces one full deserialization per call. + /// + private static readonly object EXPERIMENT_CACHE_LOCK = new object(); + private static string _experimentCacheKey; + private static IReadOnlyDictionary _experimentCacheValue; + + private static IReadOnlyDictionary LoadExperimentRecordsCached( + string path, FdrScoresSidecar.Pass pass) + { + string key = null; + if (!string.IsNullOrEmpty(path)) + { + var info = new FileInfo(path); + // A missing file gets no cache entry: LoadExperimentRecordsFrom owns that + // degrade, and caching "absent" would outlive the write that fixes it. + if (info.Exists) + { + key = string.Format(@"{0}|{1}|{2}|{3}", path, info.Length, + info.LastWriteTimeUtc.Ticks, (int)pass); + } + } + if (key == null) + return LoadExperimentRecordsFrom(path, pass); + lock (EXPERIMENT_CACHE_LOCK) + { + if (string.Equals(_experimentCacheKey, key, StringComparison.Ordinal)) + return _experimentCacheValue; + } + var loaded = LoadExperimentRecordsFrom(path, pass); + lock (EXPERIMENT_CACHE_LOCK) + { + _experimentCacheKey = key; + _experimentCacheValue = loaded; + } + return loaded; } /// From 293ade817207ceb2964b2df83ffed4b5192217b3 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 21:28:23 -0700 Subject: [PATCH 17/30] Added P16: a report is a derived view, not a phase's private output * Stated that the diagnostics report must be derivable from a pass's own sidecars alone, by a task that runs no analysis * Named the three costs of welding a report to its producing phase, one of which is inheriting that phase's memory shape instead of the reduction's * Required the two passes be independently derivable See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/docs/00-pipeline-architecture.md | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index 8697600959..9ffd1fab41 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -533,6 +533,41 @@ somewhere else now. Restoring the original path with a junction is the cheap fix that adds a second path to a hash removes relocatability for every run, not just entrapment ones. +**P16. A report is a DERIVED VIEW over the artifacts, never an output only its producing +phase can make.** Everything the diagnostics report says about a pass is a reduction over +that pass's own sidecars, so the report must be derivable from those sidecars ALONE, after +the fact, by a task that runs no analysis. Concretely, this has to work and has to stay +working: + +> Run an analysis WITHOUT `--model-diagnostics`. Then run `--task ModelDiagnostics` on the +> finished output directory. It produces the full report - both passes - by reading the +> sidecars for the pass in question, and **re-runs no analysis of any kind**. + +A pass task may fold its own half opportunistically while the data is already in hand, and +should: it is free there. But that is an OPTIMIZATION, not the definition, and it must never +become the only route. When a report can only be produced by the phase that computed the +numbers, three costs follow: + +* **Asking for the report costs a re-analysis.** The request is a reduction over files that + are already on disk, but it is priced as the phase that produced them. A cohort that + finished without the flag then pays hours to be described. +* **The report inherits the memory shape of the phase, not of the reduction.** A view over + sidecars is O(distinct) by construction; a report welded to a phase inherits whatever that + phase holds resident. That is how a page of summary statistics can put a cohort out of + reach on a fixed-memory box - the report's own footprint is not the binding constraint, + the phase it is attached to is. +* **The report cannot be regenerated after a format or presentation change.** The page's + evolution becomes gated on compute nobody should have to spend, so it stops evolving. + +Because each half is a reduction over its OWN pass's sidecars, the two passes are +independently derivable and must be independently derivable - a missing pass-1 product is +not a reason to refuse the pass-2 half, or the reverse. + +The test that proves P16 is not "the report appears": a re-analysis produces the right +report too, silently and slowly. It is that the run **does no analysis** - assert the marker +line naming the path taken, and hold the wall clock and memory band to the reduction's +O(distinct) shape rather than the phase's. + --- ## The sidecar file contract From 1d5887804dd6157add027449c73c18034f34e159 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 21:40:50 -0700 Subject: [PATCH 18/30] Reframed P16 around the streamlined-then-diagnostics resume * Stated the requirement as re-running with the flag doing only diagnostics work, which is P15's forward scan rather than a special task * Added the corollary that diagnostics work belongs in the FDR tasks, since anything captured in a fan-out task is lost to the pay-later path * Made completeness half of the test: the same report, not just a report See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/docs/00-pipeline-architecture.md | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index 9ffd1fab41..3897730ebb 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -535,13 +535,31 @@ entrapment ones. **P16. A report is a DERIVED VIEW over the artifacts, never an output only its producing phase can make.** Everything the diagnostics report says about a pass is a reduction over -that pass's own sidecars, so the report must be derivable from those sidecars ALONE, after -the fact, by a task that runs no analysis. Concretely, this has to work and has to stay -working: - -> Run an analysis WITHOUT `--model-diagnostics`. Then run `--task ModelDiagnostics` on the -> finished output directory. It produces the full report - both passes - by reading the -> sidecars for the pass in question, and **re-runs no analysis of any kind**. +that pass's own durable artifacts, so the report must be derivable from those artifacts +ALONE, after the fact, without re-running the analysis that produced them. Concretely, this +has to work and has to stay working: + +> Run an analysis WITHOUT `--model-diagnostics` - the streamlined run. Later, re-run the +> same analysis WITH `--model-diagnostics`. The ONLY work performed is producing the +> diagnostics artifacts and the HTML. No primary analysis re-runs, and **the report is +> complete** - identical to the one the flag would have produced first time. + +This is the ordinary resume model (P15) applied to the diagnostics products: they are +declared outputs like any other, so on a completed run they are the only outstanding ones +and the forward scan produces just them. It is not a special mode, and it should not need a +special task. + +**The corollary that is easy to miss: diagnostics work belongs in the FDR tasks, never in +the fan-out tasks.** A fan-out task writes the per-run artifacts; the FDR task reduces them +into the report. A diagnostic captured only in a fan-out task's memory is LOST to the +pay-later path by construction, because that path's entire premise is that the fan-out does +not re-run. The failure is silent and it degrades rather than fails: the artifact is on +disk, nothing reads it back, and the page renders without that view and says nothing. A +report that is quietly missing a panel is worse than one that is slow, because the reader +cannot tell absence from emptiness. + +The completeness half of the requirement is therefore as binding as the no-re-analysis half. +"It produced a report" is not the test; "it produced the SAME report" is. A pass task may fold its own half opportunistically while the data is already in hand, and should: it is free there. But that is an OPTIMIZATION, not the definition, and it must never @@ -563,10 +581,16 @@ Because each half is a reduction over its OWN pass's sidecars, the two passes ar independently derivable and must be independently derivable - a missing pass-1 product is not a reason to refuse the pass-2 half, or the reverse. -The test that proves P16 is not "the report appears": a re-analysis produces the right -report too, silently and slowly. It is that the run **does no analysis** - assert the marker -line naming the path taken, and hold the wall clock and memory band to the reduction's -O(distinct) shape rather than the phase's. +The test that proves P16 has two halves, and neither alone is sufficient: + +* **No analysis ran.** A re-analysis produces the right report too, silently and slowly, so + the artifact cannot distinguish them - assert the marker line naming the path taken, and + hold the memory band to the reduction's O(distinct) shape rather than the phase's. Note + that a compliant fold still READS every run's artifacts; "no analysis" means no + recomputation, not no I/O, so wall clock separates a fold from a join only by a factor. +* **The report is complete.** Byte-compare it against the report the same analysis produces + when the flag is passed up front. Any view that is present in one and absent in the other + is a diagnostic that some phase is holding privately. --- From 34ea446f3aa079a035feb1202c40d1a820e5c62d Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Mon, 7 Sep 2026 22:54:07 -0700 Subject: [PATCH 19/30] Removed the pass-2 retrain and made the report a derived view * The retrain fallback in ComputePass2Resident is now an error naming #4484. It shipped output by the method the project removed for correctness, on a warning, where the standing rule is hard-fail * Deleted the projection 2nd-pass branch, unreachable since the retrain went, and the !config.ModelDiagnostics term that read as though the flag reroutes the analysis * pass2Contributions is null in every configuration, so it deletes end to end. BuildPass2 keeps the parameter: its cards are re-sourced from the frozen pass-1 model next, and the contract stays under test * SecondPassFDR folds its diagnostics product from a completed second pass, symmetric with pass 1, ahead of the marker wipe that would clear the stamps entitling it to adopt that pass * --task ModelDiagnostics invokes whichever folds are missing instead of refusing and naming a producer, which is P15's resume applied to the diagnostics outputs (P16) * Resolved the pass-2 experiment records once per overlay instead of once per run per pass, lazily so the join still reads them after the competition See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../ModelDiagnostics/ModelDiagnosticsData.cs | 51 +- .../Osprey/Osprey.Tasks/FirstPassFdrTask.cs | 28 +- .../ModelDiagnosticsReport.cs | 47 +- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 463 +++++------------- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 156 +++++- pwiz_tools/Osprey/Osprey.Test/IOTest.cs | 143 ------ .../Osprey.Test/ModelDiagnosticsDataTest.cs | 22 +- .../Osprey/Osprey.Test/Pass2FdrSidecarTest.cs | 196 +------- pwiz_tools/Osprey/Osprey/Program.cs | 64 ++- 9 files changed, 414 insertions(+), 756 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.cs b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.cs index 60ecc9bd6a..18af358242 100644 --- a/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.cs +++ b/pwiz_tools/Osprey/Osprey.FDR/ModelDiagnostics/ModelDiagnosticsData.cs @@ -100,10 +100,10 @@ public sealed partial class ModelDiagnosticsData /// once. Null on a single-pass run (no reconciliation), which hides the switch. /// Its structural half ( / /// / ) - /// is present only when the second pass RETRAINED Percolator; under - /// OSPREY_PASS2_QVALUE=transfer it is null and the report's structural - /// cards degrade to a "pass-2 model n/a" note, while the q-driven half still - /// renders. Built by the end-of-run writer (SecondPassFdrTask) via + /// needed a retrained second-pass model, and no surviving mode trains one + /// (issue #4484), so it is null and the report's structural cards degrade to a + /// "pass-2 model n/a" note while the q-driven half still renders. Built by the + /// end-of-run writer (SecondPassFdrTask) via /// . Shares with pass 1 /// (same standardized bins). /// @@ -177,16 +177,20 @@ public sealed class ModelPass /// The complete pass-2 (final reported pool) bundle behind the report's /// top-level Pass 1 / Pass 2 switch: every pass-dependent card recomputed on /// the post-compaction, second-pass-q-valued pool. Split into a STRUCTURAL half - /// (score/model-derived) and a Q-DRIVEN half (reported-pool q-derived), because - /// the two become available under different second-pass modes: - /// - /// Retrain (OSPREY_PASS2_QVALUE=percolator): a second Percolator - /// model exists, so BOTH halves are built. - /// Confidence transfer (OSPREY_PASS2_QVALUE=transfer): no retrained - /// model, so the structural half is null (the report's Model / Density / - /// Competition cards show a "pass-2 model n/a" note) while the q-driven half -- - /// which needs only the transferred q -- still renders. - /// + /// (score/model-derived) and a Q-DRIVEN half (reported-pool q-derived). + /// + /// The Q-DRIVEN half is what every surviving second-pass mode produces: + /// transfer and protein-compact both yield reported q-values and + /// nothing else. The STRUCTURAL half needed a retrained second-pass model, and + /// there is no longer a mode that trains one - the retrain was removed because a + /// compacted pool is decoy-depleted and retraining on it mis-estimates the null + /// (issue #4484). So the structural half is null in production today and the + /// report's Model / Density / Competition cards show their "pass-2 model n/a" + /// note. It is kept as a shape rather than deleted because its replacement is + /// specified and sourced differently: the FROZEN pass-1 coefficients, already on + /// disk in .1st-pass.model.json, plus per-feature running sums folded per + /// run. still accepts contributions so that shape stays + /// covered and has somewhere to reconnect. /// /// /// Which passes this page represents and how much of the cohort reached them, so the @@ -703,10 +707,12 @@ public static ModelDiagnosticsData Build( /// from the searched library exactly as pass 1 (see ModelDiagnosticsReport). /// /// The STRUCTURAL half (Model / DensityRatio / WinFraction) is built only when - /// the second pass RETRAINED Percolator ( - /// non-null); under OSPREY_PASS2_QVALUE=transfer there is no retrained - /// model, so it stays null and the report's Model / Density / Competition cards - /// degrade to a "pass-2 model n/a" note. The Q-DRIVEN half (FdpViews / IdYield / + /// is non-null. Production passes null + /// today: it required a retrained second-pass model and no surviving mode trains + /// one (issue #4484), so those cards degrade to a "pass-2 model n/a" note. The + /// parameter and the cards are kept because their replacement is specified - + /// frozen pass-1 coefficients plus per-feature running sums - and because the + /// non-null contract stays under test. The Q-DRIVEN half (FdpViews / IdYield / /// CrossRun / PerFile) is always built from the reported pool (FdpViews is empty /// when the pool carries no entrapment). /// @@ -789,10 +795,11 @@ public static Pass2Data BuildPass2( runFdr, fdrLevel, 2, true, stratumBaseIds); progress.Report(++cardIdx); - // Structural half: only when the second pass retrained on the reported pool. Null - // contributions (transfer mode) leave Model, DensityRatio and WinFraction null and - // the report's structural cards show their n/a note. Takes the reduction computed - // above; it used to recompute a bit-identical one from the same inputs. + // Structural half: only when contributions were supplied. Production supplies none + // (there is no retrained second pass any more, #4484), which leaves Model, + // DensityRatio and WinFraction null and shows the report's n/a note. Takes the + // reduction computed above; it used to recompute a bit-identical one from the + // same inputs. pass2.Model = BuildModelPass2(pass2Contributions, precs); if (pass2.Model != null) { diff --git a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs index ce696831e2..26a17e0ae5 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs @@ -252,16 +252,6 @@ public override IEnumerable Outputs(PipelineContext ctx) } } - /// - /// True when the pass-1 diagnostics product is the single declared output this task - /// still owes: it is absent, and every other declared output exists with a current - /// validity stamp. The condition Run's fold arm turns on. - /// - /// Asked over rather than a hand-listed set, so a future - /// output is covered without anyone remembering to add it here - the failure direction - /// of a forgotten entry is then a redundant recompute rather than a wrongly-adopted - /// first pass. - /// /// /// Produce the pass-1 diagnostics product and nothing else, from a first pass that is /// already complete on disk. Streams each run's pre-compaction rows into the report @@ -273,10 +263,10 @@ public override IEnumerable Outputs(PipelineContext ctx) /// capture. Those are what a rescore needs; a report needs none of them, and retaining /// the survivors is what put a 446-run fold over a 63.7 GB box at run 266. /// - /// Reached from --task FirstPassFDR --model-diagnostics, which is the - /// supported way to give a completed analysis the pass-1 product it was run without. - /// NOT from --task ModelDiagnostics: that task renders and never processes, so - /// when this product is missing it names this one as the producer and stops. + /// Reached from --task FirstPassFDR --model-diagnostics, and from + /// --task ModelDiagnostics, which no longer refuses a missing product and names + /// this task as its producer - it falls into the ordinary pipeline, where this arm is + /// what "run FirstPassFDR" means once the first pass is already complete (P16). /// private bool FoldDiagnosticsOnly(PipelineContext ctx) { @@ -347,6 +337,16 @@ private bool FoldDiagnosticsOnly(PipelineContext ctx) return true; } + /// + /// True when the pass-1 diagnostics product is the single declared output this task + /// still owes: it is absent, and every other declared output exists with a current + /// validity stamp. The condition Run's fold arm turns on. + /// + /// Asked over rather than a hand-listed set, so a future + /// output is covered without anyone remembering to add it here - the failure direction + /// of a forgotten entry is then a redundant recompute rather than a wrongly-adopted + /// first pass. + /// private bool OnlyDiagnosticsProductOutstanding(PipelineContext ctx) { string diagnosticsPath = ModelDiagnosticsReport.Pass1SidecarPath(ctx.Config); diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs b/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs index 5eb6321288..8a49961360 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ModelDiagnostics/ModelDiagnosticsReport.cs @@ -217,7 +217,6 @@ public static void WriteFromAccumulator( /// public static void WritePass2AndFinalize( IReadOnlyList>> perFileEntries, - FeatureContributions pass2Contributions, IReadOnlyDictionary libraryById, OspreyConfig config, Action logInfo, @@ -239,9 +238,11 @@ public static void WritePass2AndFinalize( // Build the complete pass-2 (final reported pool) bundle -- every // pass-dependent card recomputed on this post-compaction, second-pass // q-valued pool -- so the page's top-level Pass 1 / Pass 2 switch can - // re-source the whole page. The structural half is null under - // confidence-transfer mode (pass2Contributions == null); the q-driven - // half is always built (FdpViews empty without an entrapment pool). + // re-source the whole page. The structural half is null in every surviving + // mode: with the retrain removed (#4484) there is no retrained pass-2 model + // to describe, which is why null is passed as the contributions rather than + // sourced. The q-driven half is always built (FdpViews empty without an + // entrapment pool). // These two steps shared a 71 s silence on the 82-file SEA-AD run of 2026-08-14, // between the classification's [ENTRAPMENT] line and "finalized report" (#4571). // BuildPass2 owns essentially all of it and carries its own per-card @@ -251,7 +252,7 @@ public static void WritePass2AndFinalize( // Measured 2026-08-15: the render that follows completes inside the same second // it starts, so it gets no line at all. data.Pass2 = ModelDiagnosticsData.BuildPass2( - perFileEntries, pass2Contributions, classByBaseId, pairByBaseId, + perFileEntries, null, classByBaseId, pairByBaseId, entrapmentRatio, config.RunFdr, config.FdrLevel, BuildPrecursorMzLookup(libraryById), stratumBaseIds); @@ -293,7 +294,6 @@ public static void WritePass2AndFinalizeFromAccumulator( ModelDiagnosticsData data, ModelDiagnosticsData.Accumulator accumulator, ModelDiagnosticsData.CoAssignmentData coAssignment, - FeatureContributions pass2Contributions, OspreyConfig config, Action logInfo, string validityKey = null) @@ -302,7 +302,9 @@ public static void WritePass2AndFinalizeFromAccumulator( { if (data == null) return; - data.Pass2 = accumulator.BuildPass2(pass2Contributions, coAssignment); + // Null contributions for the reason the resident sibling passes null: no + // surviving pass-2 mode retrains, so there is no pass-2 model to describe. + data.Pass2 = accumulator.BuildPass2(null, coAssignment); FinalizePass2(data, config, validityKey, logInfo); } catch (Exception ex) @@ -418,6 +420,37 @@ public static bool HasCompletedFirstPass(OspreyConfig config) return !string.IsNullOrEmpty(path) && File.Exists(path); } + /// + /// Whether every diagnostics product this analysis is CAPABLE of having is already on + /// disk - so the report can be re-rendered from products alone and nothing needs + /// folding. False means at least one half is outstanding and the pass that owns it has + /// to produce it. + /// + /// "Capable of" is what keeps this from being a permanent false: an analysis that + /// has finished its first pass and not its second cannot have a pass-2 product, and + /// demanding one would send every such run into a fold that has nothing to fold. So the + /// pass-2 half is required only once says there is + /// a second pass to describe. + /// + /// The pass-1 half is asked as "present AND describes THIS analysis", the same + /// two questions asks, because a product left over + /// from a different library or parameter set is not a product this run may render - it + /// has to be rebuilt, which is a fold. + /// + public static bool AllProductsCurrent(OspreyConfig config) + { + string pass1Path = ResolvePass1SidecarPath(config); + if (string.IsNullOrEmpty(pass1Path) || !File.Exists(pass1Path) || + !DescribesTheFirstPassOnDisk(config, pass1Path)) + { + return false; + } + if (!HasCompletedSecondPass(config)) + return true; + string pass2Path = ResolvePass2SidecarPath(config); + return !string.IsNullOrEmpty(pass2Path) && File.Exists(pass2Path); + } + /// /// Whether the SECOND pass completed for this configuration, asked of its own /// analysis-wide experiment sidecar rather than of the diagnostics product. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index 166032bac5..8f70565af0 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -70,14 +70,7 @@ internal static class Pass2FdrSidecar /// are the owning task's identity, /// stamped into each inline per-file validity sidecar. /// - /// - /// The second-pass Percolator model's when - /// this call actually retrained (feature histograms included if - /// --model-diagnostics), for the model-diagnostics pass-2 model view; - /// null when the 2nd-pass scores were rehydrated from sidecars (no retrain) or - /// the method is not Percolator. - /// - internal static FeatureContributions ComputeAndPersist( + internal static void ComputeAndPersist( PipelineContext ctx, bool anyRescoreWork, RescoredEntries rescored, @@ -91,10 +84,9 @@ internal static FeatureContributions ComputeAndPersist( // reads the milestone) and every other path (which reads the buffer) cannot be // handed different pools - but read LAZILY, because on the frozen path nothing // below asks for it and .Value is the O(runs x entries) build this stage exists to - // stop paying. The retrain-shaped paths (the resident 2nd pass, the projection - // sink's per-file protein-q map) genuinely index the whole pool and still get it. + // stop paying. The resident 2nd pass genuinely indexes the whole pool and still + // gets it. List>> Pool() => rescored.Value; - FeatureContributions pass2Contributions = null; // OSPREY_PASS2_QVALUE selects how this 2nd pass assigns reported q-values. // Log the active mode once so a run's provenance is in the log. An unrecognized @@ -108,53 +100,7 @@ internal static FeatureContributions ComputeAndPersist( OspreyEnvironment.PASS2_QVALUE_TRANSFER)); } - // Frozen 2nd-pass modes need the trained 1st-pass model. On a distributed - // --task SecondPassFDR node (or any resume that skipped 1st-pass training) - // it was never published in-process; reload it from the per-file sidecar and - // publish so the frozen dispatch below finds it instead of fail-fasting. No-op - // when the model is already present, the mode is the default retrain, or the - // sidecar is absent (the existing fail-fast then applies). - // protein-compact needs the ProteinCompactStratum too; it rides in the same - // sidecar, so one reload serves both surviving modes. - bool wantsFrozenModel = OspreyEnvironment.Pass2TransferQ || - OspreyEnvironment.Pass2ProteinCompact; - if (wantsFrozenModel && !ctx.TryGet(out _)) - { - var reloaded = FirstPassModelIO.LoadFromAny(perFileParquetPaths); - if (reloaded != null) - { - // ExperimentAgg is what the TRAINING process ran under (null on a sidecar - // written before the field existed). This node's own OSPREY_EXPERIMENT_AGG - // says nothing about it, so carry the recorded value rather than re-reading. - ctx.Publish(new FirstPassPercolatorModel - { Results = reloaded.Model, ExperimentAgg = reloaded.ExperimentAgg }); - ctx.LogInfo(string.Format( - @"Reloaded persisted 1st-pass model sidecar for frozen 2nd-pass (pass-1 " + - @"experiment aggregation: {0}).", - reloaded.ExperimentAgg ?? @"not recorded")); - - // Only publish a stratum the sidecar actually carried. Leaving it absent - // keeps the existing fail-fast, which is the honest outcome: an empty - // stratum would silently constrain the competition to nothing. - if (OspreyEnvironment.Pass2ProteinCompact && reloaded.StratumBaseIds != null && - !ctx.TryGet(out _)) - { - ctx.Publish(new ProteinCompactStratum(reloaded.StratumBaseIds)); - ctx.LogInfo(string.Format( - @"Reloaded the persisted protein-compact stratum ({0} base ids).", - reloaded.StratumBaseIds.Count)); - } - } - } - - // When the projection 2nd-pass compute ran (flag on), this holds the scored - // FdrProjectionSet -- non-null is the flag that the StreamingSink already - // wrote each file's .2nd-pass.fdr_scores.bin + validity sidecar DURING the - // score pass (issue #4355 struct-shrink S0 / C1: the q-values are never - // stored on the projection). The resident write block below is then only - // for the flag-off / resume path. Null on the resident path (flag off) and - // on the skip / resume path. (#4374) - FdrProjectionSet pass2Projections = null; + EnsureFrozenFirstPassPublished(ctx, perFileParquetPaths); // The frozen-model COMPETITION modes - transfer-compete, and protein-compact unless // it was told to retrain. These own their whole per-file cycle (materialize, score, @@ -310,78 +256,33 @@ internal static FeatureContributions ComputeAndPersist( var swPass2 = Stopwatch.StartNew(); + // Two modes, two shapes, and no third: OSPREY_PASS2_QVALUE normalizes to + // protein-compact or transfer and nothing else. + // // protein-compact re-scores with the frozen 1st-pass model over the // protein stratum. It owns the whole per-file cycle: materialize, score, - // compete, write the sidecar, drop - so it needs neither the projection - // engine nor a resident pool. + // compete, write the sidecar, drop - so it needs no resident pool. // - // --model-diagnostics needs the resident 2nd-pass model: its feature - // contributions feed the pass-2 model view, and the projection 2nd pass - // streams through a sink and produces none. Route --model-diagnostics to - // the resident path so ComputePass2Resident can return the model. Off the - // default output path, so byte-identity is unaffected (#4377). - // transfer takes the resident path too: it needs each survivor's RECONCILED + // transfer takes the resident path: it needs each survivor's RECONCILED // features on entry.Features, which ComputePass2Resident does. + // + // A projection 2nd pass stood between these two until the retrain was + // removed. Its condition required !Pass2TransferQ, which only the retrain + // modes could satisfy, so it had been unreachable since ad4ef8d106; the + // condition also carried a !config.ModelDiagnostics term that read as + // though the flag reroutes the analysis. It does not, and never did after + // the retrain went - the term's only remaining effect was on the reader. if (frozenCompetition) { pass2SidecarsWritten = ComputePass2FrozenCompetition( ctx, rescored, perFileParquetPaths, config, pass2Writer); } - else if (OspreyEnvironment.UseFdrProjection && config.FdrMethod.UsesPercolatorFramework() && - !config.ModelDiagnostics && !OspreyEnvironment.Pass2TransferQ) - { - // Projection 2nd pass (issue #4374 + #4355 struct-shrink S0 / C1): - // stream the reconciled PIN features through the SAME projection - // engine the 1st pass uses, rather than loading every survivor's - // 21-feature vector resident. The lean projection no longer stores - // the q-values (2nd-pass peak 80 -> 32 B); a StreamingSink assembles - // each .2nd-pass.fdr_scores.bin record DURING the score pass (from - // the streamed q-values + the survivor's ExperimentProteinQvalue looked up - // by entry_id) and flushes the per-file sidecar + validity sidecar - // directly, so the resident write block below is skipped for this - // path. The existing entry_id overlay still carries the 2nd-pass - // q-values onto the resident survivor buffer afterward (unchanged). - - // Survivor ExperimentProteinQvalue by entry_id, per file: the value - // BuildFromEntries used to carry onto the struct. All survivors - // sharing an entry_id share a precursor (hence a ModifiedSequence, - // hence a experiment_protein_qvalue), so the last-write map is exact. - var survivorsByFile = - new Dictionary>(StringComparer.Ordinal); - foreach (var kvp in Pool()) - survivorsByFile[kvp.Key] = kvp.Value; - - IReadOnlyDictionary ResolveProteinQ(string fileName) - { - var map = new Dictionary(); - if (survivorsByFile.TryGetValue(fileName, out var survivors)) - { - foreach (var e in survivors) - map[e.EntryId] = e.ExperimentProteinQvalue; - } - return map; - } - - // Per-file flush: write the .2nd-pass.fdr_scores.bin from the - // assembled records, sourced from records instead of the resident - // buffer. The write body itself - resume skip, validity sidecar, - // tallies - is the writer's, shared with every other path that emits - // one of these files. - void FlushPass2File(string fileName, IReadOnlyList records) - { - pass2Writer.Write(fileName, records); - } - - pass2Projections = ComputePass2Projection( - ctx, Pool(), perFileParquetPaths, config, - ResolveProteinQ, FlushPass2File); - } else { - // Resident 2nd pass (flag off): the byte-identity oracle. Reload - // every survivor's PIN features resident, then run the resident - // Percolator over the full FdrEntry survivor buffer. - pass2Contributions = ComputePass2Resident(ctx, Pool(), perFileParquetPaths, config); + // Resident 2nd pass: reload every survivor's PIN features resident, + // then transfer the first-pass confidence over the full FdrEntry + // survivor buffer. Throws if the transfer cannot be made. + ComputePass2Resident(ctx, Pool(), perFileParquetPaths, config); } swPass2.Stop(); ctx.LogInfo(string.Format( @@ -448,11 +349,9 @@ void FlushPass2File(string fileName, IReadOnlyList records) // sidecar) for every file that completed. // Resident / resume path only: write each file's .2nd-pass sidecar from - // the resident survivor buffer. The projection path (pass2Projections - // != null, issue #4355 struct-shrink S0 / C1) and the frozen competition - // (#4486) both wrote the .bin + validity sidecar per file as they went, so - // this loop is skipped for them - only the shared tallies they updated - // drive the summary log below. + // the resident survivor buffer. The frozen competition (#4486) wrote the + // .bin + validity sidecar per file as it went, so this loop is skipped + // for it - only the shared tallies it updated drive the summary log below. // // NOT ON THE STREAMED POOL, and this is a correctness skip rather than a // memory one. It is reachable there on the DEFAULT mode: with @@ -472,7 +371,7 @@ void FlushPass2File(string fileName, IReadOnlyList records) // so - and "not recomputed" means every one of them is already current on // disk. P13's never-conditionally-write rule binds the artifact's OWNER, and // that is not this task here. - if (pass2Projections == null && !pass2SidecarsWritten && !rescored.Streams) + if (!pass2SidecarsWritten && !rescored.Streams) { // Per-file progress: this writes one .2nd-pass.fdr_scores.bin per file // (~4.8 GB across 82) and was silent, which with the reload loop below is @@ -536,8 +435,51 @@ void FlushPass2File(string fileName, IReadOnlyList records) { ReloadPass2Sidecars(ctx, pass2Writer, Pool(), @"post-write"); } + } - return pass2Contributions; + /// + /// Publish the trained 1st-pass model - and, under protein-compact, the stratum that + /// rides in the same sidecar - from disk when this process did not train it. + /// + /// Needed by every entry point that reads frozen 1st-pass state without having + /// run the first pass: a distributed --task SecondPassFDR node, any resume that + /// skipped training, and the pass-2 diagnostics-only fold, which runs no second pass at + /// all and still has to know the stratum in order to split the acceptance boundary the + /// same way the join did. Shared rather than repeated because a fold that resolved the + /// stratum differently from the join would describe a different pool while looking like + /// the same report. + /// + /// A no-op when the model is already published or the sidecar is absent; in the + /// latter case the caller's own fail-fast applies. Only a stratum the sidecar actually + /// carried is published - an empty one would silently constrain the competition to + /// nothing, which is worse than the honest absence. + /// + internal static void EnsureFrozenFirstPassPublished( + PipelineContext ctx, IReadOnlyDictionary perFileParquetPaths) + { + if (ctx.TryGet(out _)) + return; + var reloaded = FirstPassModelIO.LoadFromAny(perFileParquetPaths); + if (reloaded == null) + return; + // ExperimentAgg is what the TRAINING process ran under (null on a sidecar + // written before the field existed). This node's own OSPREY_EXPERIMENT_AGG + // says nothing about it, so carry the recorded value rather than re-reading. + ctx.Publish(new FirstPassPercolatorModel + { Results = reloaded.Model, ExperimentAgg = reloaded.ExperimentAgg }); + ctx.LogInfo(string.Format( + @"Reloaded persisted 1st-pass model sidecar for frozen 2nd-pass (pass-1 " + + @"experiment aggregation: {0}).", + reloaded.ExperimentAgg ?? @"not recorded")); + + if (OspreyEnvironment.Pass2ProteinCompact && reloaded.StratumBaseIds != null && + !ctx.TryGet(out _)) + { + ctx.Publish(new ProteinCompactStratum(reloaded.StratumBaseIds)); + ctx.LogInfo(string.Format( + @"Reloaded the persisted protein-compact stratum ({0} base ids).", + reloaded.StratumBaseIds.Count)); + } } /// @@ -692,9 +634,25 @@ internal static void InstallStreamedPass2Overlay( if (!rescored.Streams) return; var writer = new Pass2SidecarWriter(ctx, ctx.Config, taskName, taskValidityKey); + // Resolved ONCE for the life of the overlay, and lazily. + // + // Once, because this ran per materialization: every run, every stream pass. Where + // the scope byproduct is published that is a cheap re-read of a field, but where it + // is not - a resume, and now the pass-2 diagnostics-only fold, which runs no + // competition to publish one - it deserializes the whole analysis-wide 2nd-pass + // experiment sidecar again for each run of each pass. That is O(runs x sidecar) on + // exactly the path a fold is supposed to make cheap. + // + // Lazily, because WHEN it resolves is a correctness question, not a performance + // one. Resolving at install time on the join path would capture the records before + // the second-pass competition publishes its scope; deferring to the first + // materialization puts the read after it. On the fold path there is no competition + // and the sidecar on disk is final, so either moment gives the same answer. + var experimentRecords = new Lazy>( + () => ResolvePass2ExperimentRecords(ctx)); rescored.AddPostMaterialize((fileName, entries) => OverlayPass2SidecarOntoFile( - writer, fileName, entries, ResolvePass2ExperimentRecords(ctx), ctx.LogWarning)); + writer, fileName, entries, experimentRecords.Value, ctx.LogWarning)); } /// @@ -2345,13 +2303,18 @@ FdrExperimentRecord FinishRecord(FdrScoreRecord rec) /// FirstPassFdrTask.RunPercolatorFdr over the full survivor buffer, which /// scores it in place. /// - /// Reached by the RETRAIN modes and by transfer, which needs each survivor's - /// reconciled features on entry.Features. The frozen COMPETITION modes used to - /// enter here and return early; they now have their own entry point + /// Reached ONLY by transfer, which needs each survivor's reconciled + /// features on entry.Features. The frozen COMPETITION modes used to enter here + /// and return early; they now have their own entry point /// (), because nothing about them is resident - /// any more - they never see this buffer (#4486). + /// any more - they never see this buffer (#4486). The retrain modes that were this + /// method's other caller no longer exist (#4484). + /// + /// Throws when the confidence transfer cannot be made. There is no second-pass + /// retrain to fall back on any more, and completing the run on first-pass q-values + /// would ship an analysis whose second pass silently did not happen. /// - private static FeatureContributions ComputePass2Resident( + private static void ComputePass2Resident( PipelineContext ctx, List>> perFileEntries, IReadOnlyDictionary perFileParquetPaths, @@ -2452,7 +2415,11 @@ private static FeatureContributions ComputePass2Resident( case FdrMethod.Gbdt: // protein-compact is handled by the frozen competition before the resident // feature reload, so its score pass streams one file at a time. Only - // OSPREY_PASS2_QVALUE=transfer reaches here. + // OSPREY_PASS2_QVALUE=transfer reaches here, which is why this no longer + // tests Pass2TransferQ: NormalizePass2QValue returns transfer or + // protein-compact and nothing else, so on this path the test was always + // true and reading it as a choice invited the conclusion that some other + // mode lands here. // OSPREY_PASS2_QVALUE=transfer: instead of retraining a 2nd-pass SVM on // the decoy-depleted reconciled+compacted set (which re-derives an // anti-conservative experiment-scope q), carry the pass-1 q through and @@ -2460,31 +2427,39 @@ private static FeatureContributions ComputePass2Resident( // Each moved/gap-filled peak is re-scored with the FROZEN 1st-pass model // (its RECONCILED features are on entry.Features above) and mapped through // THAT file's own (1st-pass score -> run q) table; experiment q is left as - // the pass-1 carry. Falls through to the retrain if the flag is off or the - // frozen model was not captured. See TODO-osprey_pass2_per_run_only_qvalue. - if (OspreyEnvironment.Pass2TransferQ && - ctx.TryGet(out var frozenModel) && + // the pass-1 carry. See TODO-osprey_pass2_per_run_only_qvalue. + if (ctx.TryGet(out var frozenModel) && frozenModel?.Results != null && TransferPerRunQ(perFileEntries, config, ctx, frozenModel.Results)) { - // Transferred: no retrained 2nd-pass model in transfer mode -> no - // pass-2 SVM model view for --model-diagnostics (the pass-2 FDR - // calibration curve still renders from the transferred q-values; - // the pass-1 model view still renders too). - return null; - } - if (OspreyEnvironment.Pass2TransferQ) - { - ctx.LogWarning( - "OSPREY_PASS2_QVALUE=transfer could not transfer (frozen 1st-pass " + - "model byproduct absent); falling back to the 2nd-pass Percolator retrain."); + // Transferred. There is no retrained 2nd-pass model in any surviving + // mode, so --model-diagnostics gets no pass-2 SVM model view (the + // pass-2 FDR calibration curve still renders from the transferred + // q-values; the pass-1 model view still renders too). + return; } - // Capture the 2nd-pass model for the --model-diagnostics pass-2 model - // view (retrained on the post-reconciliation pool, #4377). Capturing - // the return value does not change what RunPercolatorFdr does, so the - // resident 2nd-pass scores stay byte-identical. - return FirstPassFdrTask.RunPercolatorFdr( - perFileEntries, config, ctx, "Second-pass"); + // The transfer could not be made, and the 2nd-pass Percolator retrain that + // used to catch this case is GONE - not disabled, removed. It was cut for a + // correctness reason: compaction leaves the reconciled pool decoy-depleted, + // so a retrain on it mis-estimates the null and re-derives an + // anti-conservative experiment-scope q (issue #4484, closed "do not + // re-open the retrain"). Falling back to it produced output by the method + // the project rejected, on a warning, which is warn-and-proceed where the + // standing rule is hard-fail: the run would finish and its q-values would + // be wrong in a direction no downstream gate looks for. + // + // Every reason the transfer declines has already been logged by the code + // that declined it - an absent or unusable frozen model, no input-file + // list, an unreadable 1st-pass experiment sidecar - so this states the + // consequence and the remedy rather than re-deriving the cause. + throw new InvalidOperationException( + @"Second-pass FDR could not transfer the first-pass confidence, and " + + @"there is no second-pass retrain to fall back on (removed for issue " + + @"#4484: the compacted pool is decoy-depleted, so retraining on it " + + @"mis-estimates the null). The preceding log line names which input " + + @"was missing; the first pass must have completed and left its model " + + @"and experiment-scope sidecars beside the analysis output. Re-run " + + @"FirstPassFDR for this cohort, then SecondPassFDR again."); // Simple / Mokapot 2nd-pass paths intentionally // not implemented yet -- the in-process pipeline's // FirstPassFdrTask.RunFdr already covers Simple, and @@ -2496,186 +2471,8 @@ private static FeatureContributions ComputePass2Resident( "Second-pass FDR: {0} is not supported in SecondPassFdrTask; " + "skipping (protein FDR will run on first-pass scores)", config.FdrMethod)); - return null; - } - } - - /// - /// Projection 2nd-pass compute (flag on, issue #4374 + #4355 struct-shrink S0): - /// build the thin from the survivor buffer with - /// each row's baked to that survivor's - /// RECONCILED parquet row (via ), then - /// run the projection FirstPassFdrTask.RunPercolatorFdr through an - /// , which ALWAYS streams the reconciled features - /// per file and streams the q-value outputs straight to the per-file - /// .2nd-pass.fdr_scores.bin via (the lean - /// projection never stores them -> 32 B). - /// supplies each row's ExperimentProteinQvalue (looked up from the resident - /// survivor by entry_id, no longer carried on the struct). Returns the scored - /// projection as the flag that the sink wrote the sidecars; the survivor buffer - /// is intentionally left unscored (the entry_id overlay carries the q-values - /// back). No full-population PercolatorEntry/PercolatorResult stack is built. - /// - private static FdrProjectionSet ComputePass2Projection( - PipelineContext ctx, - List>> perFileEntries, - IReadOnlyDictionary perFileParquetPaths, - OspreyConfig config, - Func> resolveProteinQ, - Action> flushFile) - { - // Canonicalize the survivor buffer order EXACTLY as the resident path does. - // The FdrEntry RunPercolatorFdr overload (the flag-off oracle) sorts - // perFileEntries in place by (EntryId, Charge, ScanNumber, ParquetIndex) as - // its first step (PercolatorEngine.cs) -- the post-rescore pool can carry - // gap-fill entries appended after the sorted pre-existing rows, and that - // re-sort moves them into place. Downstream Stage 7/8 (protein FDR, blib - // retention-time reporting) reads this buffer IN ORDER, so its ordering is - // byte-identity-critical even though the projection carries its own sorted - // copy. The projection path routes the buffer to the SVM as a thin copy and - // never sorts the buffer itself, so replicate the oracle's sort here or the - // gap-fill order diverges and file-level RT sums drift (issue #4374). - foreach (var kvp in perFileEntries) - { - kvp.Value.Sort(FdrEntry.CANONICAL_ORDER); // Array.Sort OK: CANONICAL_ORDER's terminal key ParquetIndex is unique per survivor here (reconciled-write numbering), so the comparison never ties - } - - // Per-file reconciled scores path (Stage 6's rescored features when a - // reconciled sibling exists, else the original Stage 4 parquet), or null when - // no parquet was mapped -- mirrors the resident reload's effective-path pick. - string Recon(string fileName) => - perFileParquetPaths.TryGetValue(fileName, out string parquetPath) - ? ParquetScoreCache.EffectiveScoresPathFromScoresPath(parquetPath) - : null; - - // identity -> reconciled row, resolved one file at a time so no more than one - // file's map is resident. On a missing parquet or a read fault, return an - // empty map -> every entry resolves to uint.MaxValue -> basic-feature - // fallback, byte-identical to the resident path (null Features -> - // BuildBasicFeatures). - IReadOnlyDictionary RowMap(string fileName) - { - string recon = Recon(fileName); - if (recon == null) - { - ctx.LogWarning(string.Format( - "Second-pass FDR: no parquet path mapped for file '{0}' " + - "(entries will run with basic-feature fallback). " + - "Check that each file's reconciled parquet is present.", fileName)); - return new Dictionary(); - } - try - { - return BuildReconciledScoreIndexToRow(recon); - } - catch (Exception ex) - { - ctx.LogWarning(string.Format( - "Second-pass FDR: failed to read identity columns from {0}: {1}", - recon, ex.Message)); - return new Dictionary(); - } - } - - var swReloadRows = Stopwatch.StartNew(); - var projections = FdrProjectionSet.BuildFromEntries(perFileEntries, RowMap); - swReloadRows.Stop(); - - // Preserve the resident path's nMapped < count visibility (risk #6): a - // survivor whose identity is absent from the reconciled parquet resolves to - // uint.MaxValue and runs on the basic-feature fallback. On the standard - // datasets every survivor maps (nMapped == count), so this warns only on a - // genuine stub/parquet mismatch. - int totalMapped = 0; - foreach (var kvp in projections.PerFile) - { - int total = kvp.Value.Count; - int nMapped = 0; - foreach (var proj in kvp.Value) - { - if (proj.ParquetIndex != uint.MaxValue) - nMapped++; - } - if (nMapped < total) - { - ctx.LogWarning(string.Format( - "Second-pass FDR: file '{0}' reconciled parquet is missing {1} of " + - "{2} survivor identities; those entries run with basic-feature " + - "fallback. Stub/parquet mismatch - check reconciled-parquet output integrity.", - kvp.Key, total - nMapped, total)); - } - totalMapped += nMapped; - } - ctx.LogInfo(string.Format( - "[TIMING] Baked reconciled rows for {0} survivor entries: {1:F1}s", - totalMapped, swReloadRows.Elapsed.TotalSeconds)); - - // Features streamed from the reconciled parquet by the baked (reconciled) - // ParquetIndex; NaN/Inf are clamped to 0 by LoadPinFeaturesFromParquet -- the - // same normalization the resident reload applied. A parquet-less file yields - // an empty row list, so ResolveFeatureRow falls back to basic features. - Func> load2 = fileName => - { - string recon = Recon(fileName); - if (recon == null) - return Array.Empty(); - return ParquetScoreCache.LoadPinFeaturesFromParquet(recon); - }; - - // The caller gates on FdrMethod.Percolator, so the projection path is only - // ever Percolator; the projection engine always streams via load2. The - // StreamingSink assembles + writes each file's .2nd-pass.fdr_scores.bin from - // the streamed q-values + the survivor's ExperimentProteinQvalue during the score - // pass, so the q-values are never stored on the projection (issue #4355 / C1). - // The EXPERIMENT-scope columns collapse into one record per entry_id, published for - // WritePass2ExperimentSidecar to finish and write after the protein FDR runs - // (format v5, issue #4486). - var experiment = new FdrExperimentAccumulator(); - var sink = new FdrStreamingSink( - projections, config, "Second-pass", resolveProteinQ, flushFile, experiment); - FirstPassFdrTask.RunPercolatorFdr( - projections, config, ctx, "Second-pass", load2, sink); - ctx.Publish(new Pass2ExperimentScope(experiment)); - return projections; - } - - /// - /// Build the reconciled parquet's (entry_id, charge, scan_number) -> row - /// map from its lean stub identity columns - /// (, which assigns - /// = row). The mirror of - /// that yields the ROW INDEX - /// instead of the feature vector: that loader keys featRows[i] by identity - /// and the streaming score pass reads rows[row] by the baked - /// , so - /// rows[map[identity]] == featByScoreIndex[identity] - the streamed feature - /// lookup is byte-identical to the resident identity binding (issue #4374 risk - /// #2). Because the reconciled parquet is written - /// (entry_id, charge, scan_number)-sorted, the row is scan-monotonic within - /// a (entry_id, charge) group, which is what keeps the scan-omitted - /// projection sort valid. Duplicate identities keep the last row (map overwrite), - /// matching the loader. Reads only the identity columns (no PIN feature / heavy - /// blob load), one file at a time. - /// - internal static Dictionary BuildReconciledScoreIndexToRow( - string reconciledPath) - { - var stubs = ParquetScoreCache.LoadFdrStubsFromParquet(reconciledPath); - var map = new Dictionary(stubs.Count); - for (int i = 0; i < stubs.Count; i++) - { - // KEY is the row's score_index - its identity. VALUE is its position in THIS - // file, because that is what addresses LoadPinFeaturesFromParquet's positional - // feature array. The two were the same number before the reconciled parquet - // became a subset, which is why this used to be able to conflate them. - if (stubs[i].ParquetIndex.HasValue) - map[stubs[i].ParquetIndex.Value] = (uint)i; + return; } - // No collision to reason about any more. This keyed on - // (entry_id, charge, scan_number) and needed a paragraph explaining why a - // duplicate identity was harmless; score_index is unique per row by construction, - // including for gap-fill rows, which are numbered past the source row count. - return map; } /// diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index d519218524..5d0dcdf965 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -264,6 +264,31 @@ public override string ValidityKey(PipelineContext ctx) public override bool Run(PipelineContext ctx) { + // The pass-2 diagnostics product is the ONLY outstanding output: every + // computational artifact this task produces is already on disk and key-current, and + // the driver reached Run solely because the pass-2 diagnostics JSON is missing - + // the state a cohort is in when it finished WITHOUT --model-diagnostics and the flag + // is added later. Fold the report from the completed second pass rather than + // re-running the join (P16: a report is a derived view over the artifacts). + // + // AHEAD OF THE MARKER WIPE BELOW, and that placement is the whole correctness + // argument. The wipe clears every declared output's validity stamp; running it + // first would destroy the record that this second pass is complete - which is the + // only evidence the fold is entitled to adopt it - and the next resume would then + // re-run the join it was just spared. Pass 1 has the same arm above its own writers + // for the same reason (FirstPassFdrTask.Run). + // + // The cost of getting this wrong is the pass-1 trap restated: a SecondPassFDR that + // genuinely re-ran produces the RIGHT report, in 69 minutes on a 446-run cohort + // instead of minutes, and no gate would ever report the difference. Only the log + // line below distinguishes them. + if (ctx.Config.ModelDiagnostics && OnlyDiagnosticsProductOutstanding(ctx)) + { + ctx.LogInfo(@"SecondPassFDR: every output but the model-diagnostics product is " + + @"current; folding the pass-2 report from the completed second pass."); + return FoldPass2DiagnosticsOnly(ctx); + } + // Mid-Run crash safety: see FirstPassFdrTask.Run for rationale. foreach (var output in Outputs(ctx)) TaskValiditySidecar.Delete(output, Name); @@ -348,10 +373,6 @@ public override bool Run(PipelineContext ctx) ReleaseUnscorableLibraryFragments(rescored, rescored.FileCount, fullLibrary, ctx); - // The 2nd-pass Percolator model, captured for the model-diagnostics - // pass-2 model view; null when no reconciliation rescore happened. - FeatureContributions pass2Contributions = null; - // Second-pass FDR. ALWAYS runs, because it always has a file to write. // // What is conditional is the RECOMPUTE, not the artifact: the second Percolator @@ -369,7 +390,7 @@ public override bool Run(PipelineContext ctx) // reconciled features, reruns Percolator, writes the .2nd-pass sidecars, and // reloads them onto the stubs so downstream protein FDR + blib see the 2nd-pass // q-values. - pass2Contributions = Pass2FdrSidecar.ComputeAndPersist( + Pass2FdrSidecar.ComputeAndPersist( ctx, AnyReconciledParquet(config), rescored, perFileParquetPaths, Name, ValidityKey(ctx)); // From here on, every fold must see the SECOND pass's answer. On the resident pool @@ -510,13 +531,13 @@ public override bool Run(PipelineContext ctx) // page, same 2nd-pass.model-diagnostics.json. if (rescored.Streams) { - WritePass2DiagnosticsStreamed(ctx, rescored, pass2Contributions, libraryById, + WritePass2DiagnosticsStreamed(ctx, rescored, libraryById, config, stratumBaseIds); } else { ModelDiagnosticsReport.WritePass2AndFinalize( - rescored.Value, pass2Contributions, libraryById, config, ctx.LogInfo, + rescored.Value, libraryById, config, ctx.LogInfo, stratumBaseIds, ValidityKey(ctx)); } } @@ -524,6 +545,121 @@ public override bool Run(PipelineContext ctx) return true; } + /// + /// True when the pass-2 diagnostics product is the single declared output this task + /// still owes: it is absent, and every other declared output exists with a current + /// validity stamp. The condition 's fold arm turns on. + /// + /// Asked over rather than a hand-listed set, so a future + /// output is covered without anyone remembering to add it here - the failure direction + /// of a forgotten entry is then a redundant re-join rather than a wrongly-adopted + /// second pass. The HTML page is the one exception: it is co-produced with the pass-2 + /// product by the same writer, so a stale or missing page is what this arm exists to + /// fix and cannot be a reason to decline. + /// + private bool OnlyDiagnosticsProductOutstanding(PipelineContext ctx) + { + string pass2Path = ModelDiagnosticsReport.Pass2SidecarPath(ctx.Config); + if (string.IsNullOrEmpty(pass2Path) || File.Exists(pass2Path)) + return false; + // A second pass that never completed has nothing to fold FROM, and adopting it + // would describe a partial cohort as a whole one. Asked of the analysis-wide + // 2nd-pass experiment sidecar, which is this task's own end-of-join output. + if (!ModelDiagnosticsReport.HasCompletedSecondPass(ctx.Config)) + return false; + string reportPath = ModelDiagnosticsReport.ReportPath(ctx.Config); + string validityKey = ValidityKey(ctx); + foreach (string output in Outputs(ctx)) + { + if (string.Equals(output, reportPath, StringComparison.OrdinalIgnoreCase)) + continue; + if (PerFileResumeDriver.IsCurrent(output, Name, validityKey)) + continue; + // Name the first output that failed. Declining here is not an error - it means + // a genuine second pass is owed - but on a large cohort it is the difference + // between minutes and over an hour, and without this line the only symptom is + // that the run takes a very long time and still produces the right answer. + ctx.LogInfo(string.Format( + @"SecondPassFDR: not folding diagnostics from completed work - {0} is {1}, " + + @"so the second pass is re-run.", + output, File.Exists(output) ? @"present but not current for this analysis" + : @"missing")); + return false; + } + return true; + } + + /// + /// Produce the pass-2 diagnostics product and nothing else, from a second pass that is + /// already complete on disk. The pass-2 sibling of + /// FirstPassFdrTask.FoldDiagnosticsOnly, and the implementation of P16 for this + /// half of the report. + /// + /// What it does NOT do is the point: no second-pass FDR, no protein FDR, no blib, + /// no FDRBench, no reconciled-parquet rewrite. Those are the JOIN, and every one of + /// their outputs is already on disk and key-current - that is what + /// established before this ran. + /// + /// "No analysis" does not mean instant, and reading it that way makes a correct + /// fold look broken. The pass-2 cards are reductions over every run's 2nd-pass sidecar + /// and reconciled parquet, and the co-assignment panel needs two reads of the pool, so + /// this streams all N runs - what it never does is RECOMPUTE any of them. The + /// distinguishing evidence is the marker line and the O(distinct) memory band, not the + /// wall clock, which separates a fold from a join by a factor rather than a category. + /// + /// Three stream passes, each rebuilding one run at a time and dropping it: the + /// experiment-q reclamp, then the accumulator fold sharing a pass with the panel's + /// cutoff phase, then the panel's detection phase. The reclamp is here and not skipped + /// because it is NOT persisted - the join applies it to the reported pool after protein + /// FDR, so a fold that omitted it would describe a pool the analysis never reported, and + /// the byte-comparison against the flag-up-front report is what would catch it. + /// + private bool FoldPass2DiagnosticsOnly(PipelineContext ctx) + { + var config = ctx.Config; + var rescored = ctx.Get(); + var libraryById = ctx.Get().Value; + var perFileParquetPaths = ctx.Get().Value; + + // The stratum, from the same sidecar the join reloads it from. Under + // protein-compact it SPLITS the pass-2 acceptance boundary in two (#4573), so a + // fold without it would draw one boundary where the join drew two and produce a + // different - complete, plausible, wrong - co-assignment panel. + Pass2FdrSidecar.EnsureFrozenFirstPassPublished(ctx, perFileParquetPaths); + HashSet stratumBaseIds = null; + if (OspreyEnvironment.Pass2ProteinCompact && + ctx.TryGet(out var pcStratum)) + { + stratumBaseIds = pcStratum.BaseIds; + } + + // The same two steps the join performs between its second pass and its report, in + // the same order, so the two routes describe the identical pool. Neither is + // analysis: the overlay applies values already on disk, and the reclamp is a fold + // over them plus a per-run apply. + Pass2FdrSidecar.InstallStreamedPass2Overlay(ctx, rescored, Name, ValidityKey(ctx)); + ReclampExperimentQToBestRun(rescored); + + ctx.LogInfo(string.Format( + @"SecondPassFDR: folding the second pass from {0} run(s), one run resident at a " + + @"time (no second-pass FDR, no protein FDR, no blib).", rescored.FileCount)); + + // One report, two survivor shapes, and the choice is the stage's existing one - + // taken here rather than re-decided, so the fold and the join cannot render from + // different arms of the same comparison. + if (rescored.Streams) + { + WritePass2DiagnosticsStreamed(ctx, rescored, libraryById, config, stratumBaseIds); + } + else + { + ModelDiagnosticsReport.WritePass2AndFinalize( + rescored.Value, libraryById, config, ctx.LogInfo, + stratumBaseIds, ValidityKey(ctx)); + } + return true; + } + /// /// Build and write the pass-2 --model-diagnostics product from the STREAMED /// survivor source, holding no more than one run's entries at a time. @@ -551,13 +687,12 @@ public override bool Run(PipelineContext ctx) /// panel and logging why is the intended outcome of those guards; losing the run is not. /// private void WritePass2DiagnosticsStreamed(PipelineContext ctx, RescoredEntries rescored, - FeatureContributions pass2Contributions, IReadOnlyDictionary libraryById, OspreyConfig config, HashSet stratumBaseIds) { try { - WritePass2DiagnosticsStreamedCore(ctx, rescored, pass2Contributions, libraryById, + WritePass2DiagnosticsStreamedCore(ctx, rescored, libraryById, config, stratumBaseIds); } catch (Exception ex) @@ -568,7 +703,6 @@ private void WritePass2DiagnosticsStreamed(PipelineContext ctx, RescoredEntries } private void WritePass2DiagnosticsStreamedCore(PipelineContext ctx, RescoredEntries rescored, - FeatureContributions pass2Contributions, IReadOnlyDictionary libraryById, OspreyConfig config, HashSet stratumBaseIds) { @@ -655,7 +789,7 @@ private void WritePass2DiagnosticsStreamedCore(PipelineContext ctx, RescoredEntr } ModelDiagnosticsReport.WritePass2AndFinalizeFromAccumulator( - pass1Data, accumulator, coAssignment, pass2Contributions, config, ctx.LogInfo, + pass1Data, accumulator, coAssignment, config, ctx.LogInfo, ValidityKey(ctx)); } diff --git a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs index f8ef53abda..69998184a6 100644 --- a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs @@ -2158,149 +2158,6 @@ public void TestHasPinFeatureColumnsRejectsFeaturelessParquet() } } - /// - /// Issue #4374 risk #2 (the highest-value new test): the 2nd-pass projection - /// bakes each survivor's ParquetIndex from - /// , then the streaming - /// score pass reads the feature row at that index. This must resolve the EXACT - /// vector the resident 2nd pass binds via - /// + - /// MapFeaturesByScoreIndex. Writes a reconciled-parquet fixture whose rows - /// arrive in NON-sorted identity order plus a gap-fill-style row that interleaves - /// into the (entry_id, charge, scan_number) sort, each carrying a DISTINCT - /// 21-feature vector, then asserts: - /// - /// featRows[rowMap[identity]] == featByIdentity[identity] for every - /// identity (risk #2) -- so the streamed feature lookup is byte-identical to the - /// resident identity binding; distinct vectors make a mis-mapping observable; - /// within each (entry_id, charge) group the baked row increases with - /// scan (risk #3) -- what keeps the scan-omitted projection sort - /// (EntryId, Charge, ParquetIndex) equal to the legacy - /// (EntryId, Charge, ScanNumber, ParquetIndex) order. - /// - /// - [TestMethod] - public void TestBuildReconciledScoreIndexToRowMatchesFeatureBinding() - { - string path = Path.GetTempFileName() + ".parquet"; - try - { - // Deliberately unsorted, with two charges of entry 100 and a second - // (later-scan) row for entry 101 that interleaves into the reconciled - // (entry_id, charge, scan_number) sort -- a gap-fill-style append. - var entryIds = new uint[] { 101, 100, 100, 101, 100 }; - var charges = new byte[] { 2, 3, 2, 2, 2 }; - var scans = new uint[] { 2200, 1500, 1100, 1200, 1300 }; - - var entries = new List(); - for (int i = 0; i < entryIds.Length; i++) - { - entries.Add(new CoelutionScoredEntry - { - EntryId = entryIds[i], - IsDecoy = false, - Sequence = "PEPTIDE", - ModifiedSequence = "PEPTIDE", - Charge = charges[i], - ScanNumber = scans[i], - FileName = "recon.mzML", - PeakBounds = new XICPeakBounds { StartRt = 4.0, EndRt = 5.0 }, - // Distinct feature vector per row so a wrong identity->row map - // surfaces as a value mismatch, not a silent pass. - Features = new CoelutionFeatureSet - { - CoelutionSum = 10.0 + i, - CoelutionMax = 20.0 + i, - NCoelutingFragments = (byte)(3 + i), - PeakApex = 100.0 + i, - PeakArea = 200.0 + i, - PeakSharpness = 0.3 + i, - Xcorr = 50.0 + i, - ConsecutiveIons = (byte)(1 + i), - ExplainedIntensity = 0.50 + i * 0.01, - MassAccuracyMean = -0.5 - i, - AbsMassAccuracyMean = 0.5 + i, - RtDeviation = 0.1 + i, - AbsRtDeviation = 0.1 + i, - Ms1PrecursorCoelution = 0.80 + i * 0.01, - Ms1IsotopeCosine = 0.90 + i * 0.01, - MedianPolishCosine = 0.88 + i * 0.001, - MedianPolishResidualRatio = 0.15 + i * 0.001, - SgWeightedXcorr = 2.3 + i, - SgWeightedCosine = 0.87 + i * 0.001, - MedianPolishMinFragmentR2 = 0.70 + i * 0.001, - MedianPolishResidualCorrelation = 0.30 + i * 0.001, - }, - }); - } - - // WriteScoresParquet re-sorts (entry_id, charge, scan_number) and assigns - // ParquetIndex = row -- exactly the reconciled write path. - ParquetScoreCache.WriteScoresParquet(path, entries, null); - - var rowMap = Pass2FdrSidecar.BuildReconciledScoreIndexToRow(path); - var featByScoreIndex = Pass2FdrSidecar.LoadReconciledFeaturesByScoreIndex(path); - var featRows = ParquetScoreCache.LoadPinFeaturesFromParquet(path); - - Assert.AreEqual(entryIds.Length, rowMap.Count); - Assert.AreEqual(entryIds.Length, featByScoreIndex.Count); - Assert.AreEqual(entryIds.Length, featRows.Count); - - // Risk #2: the score index addresses that row's own feature vector, so the - // streamed lookup equals the resident binding byte-for-byte. This file was - // written by WriteScoresParquet, which has no score_index column, so each - // row's index IS its position - which is exactly the property that lets a - // pre-#4486 reconciled parquet be read without one. - foreach (var kvp in featByScoreIndex) - { - Assert.IsTrue((int)kvp.Key < featRows.Count, "score index in range"); - CollectionAssert.AreEqual(kvp.Value, featRows[(int)kvp.Key], - "the score index must address that row's own feature vector"); - } - - // Risk #3: within each (entry_id, charge) group the reconciled row is - // scan-monotonic -- what validates the scan-omitted projection sort. - // The identity -> row lookup is built here rather than taken from a - // production map: score_index is a row IDENTITY now, not a row position, and - // conflating the two is exactly what this change removed. - var stubsForRows = ParquetScoreCache.LoadFdrStubsFromParquet(path); - var rowByIdentity = new Dictionary<(uint, byte, uint), uint>(); - for (int r = 0; r < stubsForRows.Count; r++) - { - rowByIdentity[(stubsForRows[r].EntryId, stubsForRows[r].Charge, - stubsForRows[r].ScanNumber)] = (uint)r; - } - var groups = new Dictionary<(uint, byte), List<(uint scan, uint row)>>(); - for (int i = 0; i < entryIds.Length; i++) - { - // Resolved from the written rows, not from the loop counter: the write - // re-sorts into canonical order, so input position is not row position. - uint row = rowByIdentity[(entryIds[i], charges[i], scans[i])]; - var key = (entryIds[i], charges[i]); - if (!groups.TryGetValue(key, out var list)) - { - list = new List<(uint, uint)>(); - groups[key] = list; - } - list.Add((scans[i], row)); - } - foreach (var kv in groups) - { - var list = kv.Value; - list.Sort((a, b) => a.scan.CompareTo(b.scan)); - for (int k = 1; k < list.Count; k++) - { - Assert.IsTrue(list[k].row > list[k - 1].row, - "reconciled row must increase with scan within a (entry_id, charge) group"); - } - } - } - finally - { - TryDeleteFile(path); - } - } - /// /// Verifies GetScoresPath returns the expected path. /// diff --git a/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs b/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs index 89cde0f3b7..b5d4a9f128 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ModelDiagnosticsDataTest.cs @@ -838,7 +838,15 @@ private static void TestStreamingAccumulatorMatchesBatchPass2() for (int i = 0; i < 10; i++) facc.Add(new[] { -1.0, 0.0 }, true); var contrib = facc.Build(new List { new[] { 2.0, -1.0 } }, infos); - // Batch build (the resident-path oracle). + // Batch build (the resident-path oracle), with contributions supplied. + // + // NOT a configuration production can reach: no surviving second-pass mode + // retrains, so the pipeline passes null here (issue #4484). This arm tests + // BuildPass2's CONTRACT - that a non-null contributions argument builds the + // structural half and that the streamed accumulator agrees with the batch build + // on it - which is what keeps the shape covered until its replacement source + // (frozen pass-1 coefficients plus per-feature running sums) is wired in. The + // transfer arm below is the one that mirrors what production actually runs. var batch = ModelDiagnosticsData.BuildPass2(perFileEntries, contrib, cls, pair, r, runFdr, level, mzLookup); @@ -885,9 +893,11 @@ private static void TestStreamingAccumulatorMatchesBatchPass2() Assert.IsNotNull(batch.DensityRatio); Assert.IsNotNull(batch.WinFraction); - // Transfer mode: no retrained model, so the structural half stays null on BOTH arms. - // Worth pinning separately because it is the arm protein-compact actually runs, and - // a streamed build that invented a Model there would go unnoticed by the assert above. + // Null contributions, which is EVERY production configuration: no surviving + // second-pass mode retrains, so the structural half stays null on BOTH arms. This + // is the representative arm, not a special case - a streamed build that invented a + // Model here would go unnoticed by the assert above, and this is the shape the + // pipeline actually renders. var batchT = ModelDiagnosticsData.BuildPass2(perFileEntries, null, cls, pair, r, runFdr, level, mzLookup); var accT = new ModelDiagnosticsData.Accumulator(runNames, cls, pair, r, runFdr, level, 2); @@ -908,8 +918,8 @@ private static void TestStreamingAccumulatorMatchesBatchPass2() Assert.AreEqual( JsonConvert.SerializeObject(batchT, settings), JsonConvert.SerializeObject(accT.BuildPass2(null, panelT), settings), - @"streamed pass-2 accumulator must byte-match the batch build in transfer mode"); - Assert.IsNull(batchT.Model, @"transfer mode -> no retrained model, structural half null"); + @"streamed pass-2 accumulator must byte-match the batch build with no contributions"); + Assert.IsNull(batchT.Model, @"no retrained second pass -> structural half null"); Assert.IsNull(batchT.WinFraction); // STRATIFIED, which is the only shape the streamed arm ever runs in production: diff --git a/pwiz_tools/Osprey/Osprey.Test/Pass2FdrSidecarTest.cs b/pwiz_tools/Osprey/Osprey.Test/Pass2FdrSidecarTest.cs index 878825f7b6..c9647fecce 100644 --- a/pwiz_tools/Osprey/Osprey.Test/Pass2FdrSidecarTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/Pass2FdrSidecarTest.cs @@ -23,11 +23,8 @@ using System; using System.Collections.Generic; -using System.IO; -using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using pwiz.Osprey.Core; -using pwiz.Osprey.FDR; using pwiz.Osprey.IO; using pwiz.Osprey.Tasks; @@ -38,8 +35,8 @@ namespace pwiz.Osprey.Test /// sidecar step extracted from SecondPassFdrTask.Run. Covers the pure /// seam (the /// reconciled-feature overlay) that previously rode only the nightly - /// regression, plus the increment (A) scan-omitted 2nd-pass projection sort - /// equivalence (). + /// regression, plus the OSPREY_PASS2_QVALUE=transfer score-to-q table and its + /// per-run q assignment. /// The reload/percolator/sidecar-IO orchestration itself stays parity-locked /// and is characterized by regression.ps1, not here. /// @@ -96,183 +93,6 @@ public void TestMapFeaturesByScoreIndex() Assert.AreSame(stale, loneEntry.Features); } - /// - /// Guards the byte-identity invariant behind increment (A): the scan-omitted - /// 2nd-pass projection sort key (EntryId, Charge, ParquetIndex) -- where - /// ParquetIndex is the RECONCILED-parquet row baked by - /// - must produce - /// the SAME row order as the legacy/oracle resident sort key - /// (EntryId, Charge, ScanNumber, original-ParquetIndex) (the FdrEntry - /// overload of PercolatorEngine.RunPercolatorFdr). The two provably - /// coincide for distinct scans because the reconciled parquet is written - /// (entry_id, charge, scan)-sorted, so its row is scan-monotonic within - /// a (entry_id, charge) group. The never-asserted corner -- exercised - /// here -- is the scan-tie / gap-fill case: the reconciled re-sort is a STABLE - /// OrderBy(EntryId).ThenBy(Charge).ThenBy(ScanNumber) with NO ParquetIndex - /// tiebreak, which the streaming transfer reproduces by merging gap-fill rows into - /// canonical position (ParquetScoreCache.StreamReconciledScoresParquet). Two clean 8-file - /// Carafe runs were byte-identical end-to-end but never asserted this in - /// isolation. - /// - /// The fixture packs all three risk factors into one (EntryId, Charge) - /// group: multiple distinct scans, a scan-tie (two rows sharing - /// (EntryId, Charge, ScanNumber) with different original ParquetIndex), - /// and an appended gap-fill row (the sentinel). The - /// reconciled parquet is produced by the REAL Stage-6 path -- the gap-fill is - /// merged into canonical scan position by - /// ParquetScoreCache.StreamReconciledScoresParquet and read back through - /// BuildReconciledScoreIndexToRow - so the tie/gap-fill placement is - /// production's, not a mock. The projection itself is baked by the real - /// resolver path. - /// - [TestMethod] - public void TestScanOmittedProjectionSortMatchesLegacyOrder() - { - const string fileName = @"file1"; - - // Survivor buffer (the rows both sorts operate on), in a deliberately - // scrambled construction order so neither sort is a no-op. Each row carries - // a distinct CoelutionSum marker (10..70) used only as a stable per-row token - // to compare the two resulting orders -- BuildFromEntries copies CoelutionSum - // straight onto the FdrProjection. Group A = (EntryId 100, Charge 2): - // P(scan10), G(gap-fill scan15), Q(scan20), R(scan20 == scan-tie with Q), - // S(scan30). Group B = (EntryId 200, Charge 3, decoys): T(scan5), U(scan25). - // Original ParquetIndex is (entry,charge,scan)-monotonic across the real rows - // (0..5). The gap-fill row carries 6 - the score_index the Stage 6 write assigns - // it, one past the source row count - not the uint.MaxValue sentinel it holds - // before the write. That is the state the survivor buffer is in afterwards: the - // writer numbers gap-fill rows as it emits them and mutates the entries it was - // handed, and a buffer rebuilt from the reconciled parquet reads the same value - // out of the score_index column. - var rowP = MakeSurvivor(100, 2, 10, 0, 10.0, false); - var rowQ = MakeSurvivor(100, 2, 20, 1, 20.0, false); - var rowR = MakeSurvivor(100, 2, 20, 2, 30.0, false); // scan-tie with Q - var rowS = MakeSurvivor(100, 2, 30, 3, 40.0, false); - var rowT = MakeSurvivor(200, 3, 5, 4, 50.0, true); - var rowU = MakeSurvivor(200, 3, 25, 5, 60.0, true); - var rowG = MakeSurvivor(100, 2, 15, 6, 70.0, false); // gap-fill: 6 == source row count - var survivors = new List { rowS, rowR, rowT, rowG, rowP, rowU, rowQ }; - - // Build the reconciled parquet through the REAL Stage-6 streaming path: write - // the survivor rows as the original scores parquet, then stream-transfer them - // with the gap-fill row, whose scan (15) the merge interleaves between the - // scan-10 and scan-20 rows -- exactly where the former load-all + re-sort put it. - var reconEntries = new List - { - MakeSurvivor(100, 2, 10, 0, 10.0, false), - MakeSurvivor(100, 2, 20, 1, 20.0, false), - MakeSurvivor(100, 2, 20, 2, 30.0, false), - MakeSurvivor(100, 2, 30, 3, 40.0, false), - MakeSurvivor(200, 3, 5, 4, 50.0, true), - MakeSurvivor(200, 3, 25, 5, 60.0, true), - }; - var reconGapFill = MakeSurvivor(100, 2, 15, uint.MaxValue, 70.0, false); - - string tmpStem = @"osprey_pass2sort_" + Guid.NewGuid().ToString(@"N"); - string originalPath = Path.Combine(Path.GetTempPath(), tmpStem + @".scores.parquet"); - string reconciledPath = Path.Combine(Path.GetTempPath(), tmpStem + @".scores-reconciled.parquet"); - try - { - ParquetScoreCache.WriteScoresParquet(originalPath, reconEntries, null, null, fileName); - var streamResult = ParquetScoreCache.StreamReconciledScoresParquet( - originalPath, reconciledPath, new Dictionary(), - new List { reconGapFill }, null, null, fileName, null, null, s => { }); - Assert.AreEqual(1, streamResult.NAppended, @"gap-fill row must append through the real Stage-6 path"); - - // REAL score_index -> reconciled-row map. No last-write-wins caveat any - // more: score_index is unique per row, gap-fill included. - var reconMap = Pass2FdrSidecar.BuildReconciledScoreIndexToRow(reconciledPath); - - // Legacy/oracle order: sort a fresh copy by the resident FdrEntry key. - var legacyList = new List(survivors); - legacyList.Sort(LegacyResidentComparison); - var legacyOrder = legacyList.Select(e => e.CoelutionSum).ToList(); - - // Projection order, mirroring Pass2FdrSidecar.ComputePass2Projection: - // (1) canonicalize the survivor buffer with the SAME legacy key (the - // production pre-sort), (2) build the projection with each row's - // ParquetIndex baked to its reconciled row, (3) sort by the scan-omitted - // projection key. - var perFileEntries = new List>> - { - new KeyValuePair>(fileName, new List(survivors)), - }; - perFileEntries[0].Value.Sort(LegacyResidentComparison); // ComputePass2Projection pre-sort - - var projections = FdrProjectionSet.BuildFromEntries(perFileEntries, _ => reconMap); - Assert.AreEqual(1, projections.PerFile.Count); - var projRows = projections.PerFile[0].Value; - Assert.AreEqual(survivors.Count, projRows.Count); - - // Sanity: confirm the corner is actually exercised. The gap-fill (marker - // 70) must have interleaved BY SCAN into the reconciled parquet -- its row - // falls between P's scan-10 row and Q's scan-20 row, not appended at the - // end -- and the scan-tied pair (markers 20 and 30) must collapse to the - // SAME reconciled row so the projection comparer genuinely ties on them. - var reconRowByMarker = new Dictionary(); - foreach (var p in projRows) - reconRowByMarker[p.CoelutionSum] = p.ParquetIndex; - Assert.IsTrue( - reconRowByMarker[10.0] < reconRowByMarker[70.0] && - reconRowByMarker[70.0] < reconRowByMarker[20.0], - @"gap-fill row must interleave by scan in the reconciled parquet"); - // Scan-tied rows now bake DISTINCT reconciled rows. They used to collapse onto - // one, because the map keyed on (entry_id, charge, scan_number) and a tie made - // two rows indistinguishable; keyed on score_index they are two rows, which is - // what they always were. The projection comparer still ties on them - it omits - // scan - so this test's invariant below is unaffected, and it is exercised by a - // genuine comparer tie rather than by two entries sharing a baked row. - Assert.AreNotEqual(reconRowByMarker[20.0], reconRowByMarker[30.0], - @"scan-tied rows are distinct rows and must bake distinct score indices"); - - projRows.Sort(ProjectionComparison); - var projectionOrder = projRows.Select(p => p.CoelutionSum).ToList(); - - // The guarded invariant: scan-omitted projection order == legacy order. - CollectionAssert.AreEqual(legacyOrder, projectionOrder, - @"scan-omitted projection sort diverged from the legacy resident sort; legacy=[" + - string.Join(@",", legacyOrder) + @"] projection=[" + - string.Join(@",", projectionOrder) + @"]"); - } - finally - { - if (File.Exists(originalPath)) - File.Delete(originalPath); - if (File.Exists(reconciledPath)) - File.Delete(reconciledPath); - } - } - - private static FdrEntry MakeSurvivor( - uint entryId, byte charge, uint scanNumber, uint parquetIndex, double marker, bool isDecoy) - { - return new FdrEntry - { - EntryId = entryId, - Charge = charge, - ScanNumber = scanNumber, - ParquetIndex = parquetIndex, - CoelutionSum = marker, - IsDecoy = isDecoy, - ModifiedSequence = @"PEPTIDE" + entryId, - }; - } - - // Verbatim copy of the FdrEntry-overload comparer in - // PercolatorEngine.RunPercolatorFdr (the legacy/oracle resident sort). Inlined - // because the production comparer is a private lambda inside the SVM run and - // cannot be invoked in isolation. - private static int LegacyResidentComparison(FdrEntry a, FdrEntry b) - { - int c = a.EntryId.CompareTo(b.EntryId); - if (c != 0) return c; - c = a.Charge.CompareTo(b.Charge); - if (c != 0) return c; - c = a.ScanNumber.CompareTo(b.ScanNumber); - if (c != 0) return c; - return FdrEntry.CompareParquetIndex(a.ParquetIndex, b.ParquetIndex); - } - /// /// The OSPREY_PASS2_QVALUE=transfer score->q table /// ( + @@ -434,17 +254,5 @@ public void TestAssignPerRunQCarriesExperimentQ() Assert.AreEqual(1.0, orphan.ExperimentPeptideQvalue, 1e-12); Assert.AreEqual(0.0, orphan.ExperimentAggregateScore, 1e-12); } - - // Verbatim copy of the FdrProjectionSet-overload comparer in - // PercolatorEngine.RunPercolatorFdr (the scan-omitted projection sort). Same - // isolation caveat as LegacyResidentComparison. - private static int ProjectionComparison(FdrProjection a, FdrProjection b) - { - int c = a.EntryId.CompareTo(b.EntryId); - if (c != 0) return c; - c = a.Charge.CompareTo(b.Charge); - if (c != 0) return c; - return FdrEntry.CompareParquetIndex(a.ParquetIndex, b.ParquetIndex); - } } } diff --git a/pwiz_tools/Osprey/Osprey/Program.cs b/pwiz_tools/Osprey/Osprey/Program.cs index 6c6f9c1076..d165845497 100644 --- a/pwiz_tools/Osprey/Osprey/Program.cs +++ b/pwiz_tools/Osprey/Osprey/Program.cs @@ -380,39 +380,51 @@ static int Main(string[] args) /// /// --task ModelDiagnostics: produce the report from COMPLETED analysis state and - /// never process. Returns the process exit code when the task is finished, or -1 to fall - /// through to the one case that still needs the pipeline - folding a first pass that has - /// no diagnostics product yet. + /// never re-run the analysis. Returns the process exit code when the task is finished, + /// or -1 to fall through to the pipeline when a diagnostics product still has to be + /// folded. /// /// The three states are the developer's contract. No first-pass state is an ERROR - /// rather than a partial page; a first pass with no second is the report it can honestly - /// give, with the incompleteness stated in the page; and everything present is a - /// re-render. + /// rather than a partial page; every product present is a re-render, in seconds; and a + /// missing product is FOLDED by the pass that owns it. It used to REFUSE that third + /// case and name the command the operator should run instead, which made the report an + /// output only its producing phase could make - the thing P16 forbids. /// - /// What this replaces: the task used to fall straight through to + /// Falling through is not the old behavior returning. The task used to reach /// and execute Stages 1-7 with every write suppressed by - /// . Suppressing the WRITES does not suppress - /// the WORK - it still built the whole-run survivor pool - so asking a 446-run analysis - /// to describe itself cost as much as running it, and met the same memory wall. + /// ; suppressing the WRITES does not suppress + /// the WORK, so asking a 446-run analysis to describe itself cost as much as running it + /// and met the same memory wall. What changed is the other end: both FDR tasks now + /// recognise "the diagnostics product is my only outstanding output" and fold it from + /// their own completed artifacts, so the pipeline this falls into runs two bounded + /// folds and skips everything else on its validity stamps. That is P15's ordinary + /// resume applied to the diagnostics outputs, which is what P16 says this should have + /// been all along - not a special mode, and not a special task. /// private static int RunModelDiagnosticsTask(OspreyConfig config) { - if (ModelDiagnosticsReport.TryRenderFromProducts(config, LogInfo)) - return 0; - // Nothing to render, and this task does not make it. It reports on completed work; - // producing the pass-1 product is FirstPassFDR's job, because that product is - // FirstPassFDR's declared OUTPUT. Naming the producer is the whole content of this - // message - an operator who is told "cannot" and not "run this" has to go read code. - LogError(ModelDiagnosticsReport.HasCompletedFirstPass(config) - ? "--task ModelDiagnostics: the first pass has completed but produced no " + - "diagnostics product, so there is nothing to render. That analysis was run " + - "without --model-diagnostics. Re-run it with --task FirstPassFDR " + - "--model-diagnostics to produce the pass-1 product from the completed " + - "first-pass artifacts, then run this task again." - : "--task ModelDiagnostics: no completed first-pass FDR state to describe (no " + - "analysis-wide 1st-pass experiment sidecar beside the output). Run the " + - "analysis at least as far as FirstPassFDR first."); - return 1; + // Nothing to describe. An ERROR rather than an empty page, and the doc-00 precedent + // for a missing relay input: fail with the reason, do not continue into a wrong + // answer that looks like a right one. + if (!ModelDiagnosticsReport.HasCompletedFirstPass(config)) + { + LogError("--task ModelDiagnostics: no completed first-pass FDR state to " + + "describe (no analysis-wide 1st-pass experiment sidecar beside the " + + "output). Run the analysis at least as far as FirstPassFDR first."); + return 1; + } + // Everything this analysis can have is on disk: a pure render, seconds, no pipeline. + if (ModelDiagnosticsReport.AllProductsCurrent(config)) + return ModelDiagnosticsReport.TryRenderFromProducts(config, LogInfo) ? 0 : 1; + + // A product is outstanding. Say so before the pipeline banner, because the next + // thing the log shows is task machinery and an operator needs to know it is a fold + // rather than the re-analysis this task used to refuse to start. + LogInfo("--task ModelDiagnostics: a diagnostics product is missing for this " + + "analysis; folding it from the completed artifacts. No analysis is re-run - " + + "each pass produces its own report from its own sidecars, and every other " + + "output is left as it stands."); + return -1; } /// From 7de17740d92c54f49ccc4b80e4a0e4bd136e6b83 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 00:17:09 -0700 Subject: [PATCH 20/30] Added the pay-later diagnostics gate and fixed the resident pass-2 fold * regression.ps1 mode 11 deletes both diagnostics products from a completed run and asserts the folds run, the join's markers do NOT, and the products come back identical - the P16 scenario no other leg presented * It caught a real defect on its first run: the pass-2 fold reported pass-1 q-values under a pass-2 heading whenever the pool was resident, because the streamed overlay is a no-op there and the join's compute had always stamped the pool itself * The pass-1 model/CAL views cannot survive the pay-later path at all; they are excluded BY NAME so the leg reds on any other divergence See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 24 ++ .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 14 +- pwiz_tools/Osprey/regression.ps1 | 245 ++++++++++++++++++ 3 files changed, 280 insertions(+), 3 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index 8f70565af0..cf16d9b15f 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -525,6 +525,30 @@ private static IReadOnlyDictionary ResolvePass2Experi : LoadExperimentRecords(ctx.Config, FdrScoresSidecar.Pass.SecondPass); } + /// + /// The RESIDENT sibling of : overlay every + /// file's second-pass sidecar onto a resident survivor pool. A no-op on the streamed + /// arm, where the installed per-run overlay already does it. + /// + /// Needed by the pass-2 diagnostics fold and by nothing else. Every other caller + /// arrives here having just run , which stamps the + /// second-pass values onto the resident entries as it computes them; the fold skips + /// that compute by definition, so on the resident arm its pool would otherwise still + /// carry the FIRST pass's q-values - and the report would describe pass 1 while + /// labelling it pass 2. That failure is invisible to every other check: the page is + /// complete, every card is populated, and the numbers are real, just from the wrong + /// pass. Only a byte-comparison against the flag-up-front report catches it, which is + /// why P16 makes that comparison half of the requirement. + /// + internal static void OverlayPass2OntoResidentPool( + PipelineContext ctx, RescoredEntries rescored, string taskName, string taskValidityKey) + { + if (rescored.Streams) + return; + var writer = new Pass2SidecarWriter(ctx, ctx.Config, taskName, taskValidityKey); + ReloadPass2Sidecars(ctx, writer, rescored.Value, @"diagnostics-fold"); + } + private static void ReloadPass2Sidecars( PipelineContext ctx, Pass2SidecarWriter writer, diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 5d0dcdf965..ccd05e3b2e 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -633,11 +633,19 @@ private bool FoldPass2DiagnosticsOnly(PipelineContext ctx) stratumBaseIds = pcStratum.BaseIds; } - // The same two steps the join performs between its second pass and its report, in - // the same order, so the two routes describe the identical pool. Neither is - // analysis: the overlay applies values already on disk, and the reclamp is a fold + // The same steps the join performs between its second pass and its report, in the + // same order, so the two routes describe the identical pool. None of them is + // analysis: the overlays apply values already on disk, and the reclamp is a fold // over them plus a per-run apply. + // + // BOTH arms of the overlay, and each is a no-op on the other's arm. The join needs + // only the streamed one, because on a resident pool ComputeAndPersist has just + // stamped the second-pass values onto the entries as it computed them. This fold + // skips that compute, so on the resident arm nothing would carry pass 2 onto the + // pool and the report would describe the FIRST pass under a pass-2 heading - + // complete, plausible, every card populated, and wrong. Pass2FdrSidecar.InstallStreamedPass2Overlay(ctx, rescored, Name, ValidityKey(ctx)); + Pass2FdrSidecar.OverlayPass2OntoResidentPool(ctx, rescored, Name, ValidityKey(ctx)); ReclampExperimentQToBestRun(rescored); ctx.LogInfo(string.Format( diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 91c368ea66..89e4547c81 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -86,6 +86,19 @@ what makes CanRehydrate return false so it re-runs on demand. Runs last, in the straight-through dir, since it rewrites the report there. ~14 s per dataset - it rehydrates Stages 1-5 and re-runs Stage 7 only. + mode 11 the PAY-LATER report (P16) - deletes both diagnostics products from the + completed straight-through run, leaving every analysis artifact current, + and asks for the report again. This is the one state no other leg presents: + mode 7 re-enters a run whose products are already there, so it exercises + only the RE-RENDER, and every other leg passes --model-diagnostics up front. + Both halves of the P16 test, because neither alone is sufficient. NO + ANALYSIS RAN: each pass must log the marker naming its fold, AND the markers + a genuine join emits must be absent - a re-analysis produces the RIGHT + report, so the artifact cannot tell them apart and only the log can. THE + REPORT IS COMPLETE: both products byte-compared against the ones the + flag-up-front run wrote, because a view some phase holds privately goes + missing on this path and on no other. Runs between modes 7 and 8, which is + the only window where the cohort is complete and the blib is still current. NO dependency on the sibling ai/ checkout: data acquisition, blib golden capture/compare, and the tolerance comparators all live under @@ -2807,6 +2820,238 @@ foreach ($name in $selected) { } } + # ---- mode 11: the PAY-LATER report is FOLDED, and it is the SAME report (P16) ---- + # The scenario principle P16 names, and the one no other leg presents: an analysis that + # finished WITHOUT --model-diagnostics, asked for the report afterwards. Mode 7 re-enters + # a run whose products are already on disk, so it only ever exercises the RE-RENDER; every + # other leg passes the flag up front. That is exactly the gap the developer pointed at - + # "the small-dataset gates always run the flag up front" - and it is why a pass-2 fold + # could be missing entirely while every gate stayed green. + # + # Simulated by DELETING the products from a completed run rather than by running the + # cohort twice. The state a pay-later user is in is "every analysis artifact current, no + # diagnostics product", and deleting the products produces exactly that state for a + # fraction of the wall clock - while also handing the leg its own oracle, since the + # products just deleted are what the flag-up-front run produced. + # + # BOTH halves of the P16 test, because neither alone is sufficient: + # + # NO ANALYSIS RAN. A re-analysis produces the RIGHT report, silently and slowly, so the + # artifact cannot distinguish the two - only the log can. Asserted positively (each + # pass logs the marker naming its fold) AND negatively (the markers a genuine join + # emits must be absent). At 3 files the difference is seconds and no timing check + # could see it; at 446 it is minutes against 4h46m + 69 min. + # + # THE REPORT IS COMPLETE. Byte-compared against the products the same analysis produced + # with the flag passed up front. "It produced a report" is not the test; "it produced + # the SAME report" is - a view held privately by some phase goes missing on this path + # and on no other, which is how the CAL view's loss was found. + # + # Runs after mode 7 (which rewrites the report) and BEFORE mode 8, which invalidates the + # blib: this leg needs a cohort whose every analysis artifact is still current, because + # that currency is the whole precondition for the folds it is asserting. + if ($cfg.ModelDiagnostics) { + Write-Progress-Tc "${name}: pay-later diagnostics fold (mode 11)" + $m11Issues = [System.Collections.Generic.List[string]]::new() + $m11Pass1 = Join-Path $straightDir 'output.1st-pass.model-diagnostics.json' + $m11Pass2 = Join-Path $straightDir 'output.2nd-pass.model-diagnostics.json' + # The products as the flag-up-front run wrote them. Kept OUTSIDE the run directory so + # the fingerprint below does not see them as artifacts the fold created. + $m11Ref = Join-Path (Join-Path $runRoot $name) 'mode11-reference' + New-Item -ItemType Directory -Path $m11Ref -Force | Out-Null + $m11Missing = @($m11Pass1, $m11Pass2 | Where-Object { -not (Test-Path $_) }) + if ($m11Missing.Count -gt 0) { + # Not a silent skip: this dataset carries --model-diagnostics, so both products + # are supposed to exist by now, and their absence means an EARLIER leg failed to + # produce one. Skipping quietly would report a green gate for the missing half. + $m11Issues.Add(("expected both diagnostics products from the straight-through run, " + + "but {0} is/are absent - an earlier leg did not produce it" -f + ($m11Missing -join ', '))) + } else { + foreach ($p in @($m11Pass1, $m11Pass2)) { Copy-Item $p $m11Ref -Force } + + # Delete the products AND their validity stamps. The stamp is what a later run + # reads to decide the product is current, so leaving it behind would describe a + # state no interruption can produce. + $m11Deleted = @() + foreach ($p in @($m11Pass1, $m11Pass2)) { + foreach ($f in @(Get-ChildItem ($p + '*') -ErrorAction SilentlyContinue)) { + $m11Deleted += $f.Name + Remove-Item $f.FullName -Force + } + } + Write-Host (" deleted {0} diagnostics product file(s), leaving every analysis artifact current" -f + $m11Deleted.Count) + + $m11Before = Get-DirFingerprint -Dir $straightDir + $r11 = Invoke-OspreyRun -Mzmls $inputs.Mzmls -Library $inputs.Library ` + -Resolution $cfg.Resolution -WorkDir $straightDir -LogName 'paylater.log' ` + -Spec $cfg -Manifest $inputs.Manifest -TaskName 'ModelDiagnostics' ` + -AllowNonZeroExit + Write-Host (" pay-later fold wall {0:N1}s" -f $r11.Wall.TotalSeconds) + if ($r11.ExitCode -ne 0) { + $m11Issues.Add(("--task ModelDiagnostics exited {0}; a completed analysis missing " + + "only its diagnostics products must be able to produce them" -f $r11.ExitCode)) + } + + # ORACLE 1a: each pass says it FOLDED. Substrings, not whole lines, so the + # surrounding prose can change without breaking the gate; what they pin is that + # the fold arm was entered rather than the join. + $m11Markers = @( + @{ What = 'pass-1 fold'; Pattern = 'folding the report from the completed first pass' } + @{ What = 'pass-2 fold'; Pattern = 'folding the pass-2 report from the completed second pass' }) + foreach ($mk in $m11Markers) { + $hit = @(Select-String -Path $r11.Log -Pattern $mk.Pattern -SimpleMatch ` + -ErrorAction SilentlyContinue) + if ($hit.Count -eq 0) { + $m11Issues.Add(("no {0} marker in the log ('{1}') - the report was produced " + + "by re-running the analysis, which yields the RIGHT artifact and is the " + + "failure this leg exists to catch" -f $mk.What, $mk.Pattern)) + } + } + + # ORACLE 1b: and the join did NOT run. The positive marker alone is not enough - + # one pass could fold while the other re-computes, and the artifact would still be + # correct. These are lines only genuine analysis emits. + $m11Forbidden = @( + @{ What = 'a second-pass FDR compute'; Pattern = '[STAGE-WALL] second-pass-fdr' } + @{ What = 'protein-level FDR'; Pattern = 'Running protein-level FDR' } + @{ What = 'a per-file rescore'; Pattern = 'Re-scoring file ' }) + foreach ($fb in $m11Forbidden) { + $hit = @(Select-String -Path $r11.Log -Pattern $fb.Pattern -SimpleMatch ` + -ErrorAction SilentlyContinue) + if ($hit.Count -gt 0) { + $m11Issues.Add(("{0} ran during the pay-later report ('{1}') - asking for the " + + "report re-ran the analysis, which is what P16 forbids" -f $fb.What, $fb.Pattern)) + } + } + + # ORACLE 2: the products came back, byte for byte. This is the completeness half - + # a card some phase holds privately is present in the flag-up-front product and + # absent here, and nothing else in this leg would notice. + # The pass-1 views that CANNOT survive the pay-later path today, excluded by name + # rather than by loosening the comparison. Each is captured in memory during a + # phase this path does not re-run and is never read back from disk: + # + # cal PerFileScoringTask captures it at Stage 3 and publishes it from + # memory; nothing reads the per-file .calibration.json back, and + # the shaped row is not in that file yet (a format change). + # model the feature table needs the TRAINED model's contributions; the + # fold logs "first-pass model not retrained on this run". + # featureHistEdges the per-feature histograms are built from feature vectors as + # the model trains, so they go with it. + # + # This is a KNOWN GAP, not an accepted difference: it is exactly the class P16's + # completeness half exists to expose - a diagnostic held privately by a phase, lost + # to the path whose premise is that the phase does not re-run - and it is written up + # as owed work. It is named here so the leg stays green on everything that IS + # achievable and reds the moment ANY OTHER part of pass 1 diverges, and so the + # summary line states the exclusion on every run rather than hiding it. + # TWO sets, separated because they are not the same claim. + # + # VOLATILE is a field that cannot match between any two runs and says nothing about + # completeness: generatedUtc is when the page was written. Folding it in with the + # gap below would misreport a clock reading as a lost diagnostic. + # + # UNAVAILABLE is the real gap, and all of it has ONE cause: the pass-1 model is not + # retrained on a resumed run, so everything derived from the trained model's feature + # contributions goes with it (the feature table, the per-feature histogram edges, + # the feature count, and the composite scalar) - plus the CAL view, which + # PerFileScoringTask captures in memory at Stage 3 and nothing ever reads back off + # the per-file .calibration.json. + $m11Pass1Volatile = @('generatedUtc') + $m11Pass1Unavailable = @('cal', 'model', 'featureHistEdges', 'featureCount', + 'modelComposite') + foreach ($p in @($m11Pass1, $m11Pass2)) { + $leaf = Split-Path -Leaf $p + $ref = Join-Path $m11Ref $leaf + if (-not (Test-Path $p)) { + $m11Issues.Add(("the fold did not produce {0}" -f $leaf)) + continue + } + if ($p -eq $m11Pass1) { + # Same transform on both sides, so anything the round-trip normalises + # normalises identically and only a REAL difference survives. + $stripped = foreach ($f in @($ref, $p)) { + $o = Get-Content $f -Raw | ConvertFrom-Json + foreach ($k in ($m11Pass1Unavailable + $m11Pass1Volatile)) { + if ($o.PSObject.Properties.Name -contains $k) { + $o.PSObject.Properties.Remove($k) + } + } + ConvertTo-Json $o -Depth 64 -Compress + } + if ($stripped[0] -ne $stripped[1]) { + # Its PARENT, not $runRoot: the finally block drops $runRoot, which would + # delete the only evidence of the failure it just reported. + $keep = Join-Path (Split-Path $runRoot -Parent) ('mode11-diff-' + $name) + New-Item -ItemType Directory -Path $keep -Force | Out-Null + Copy-Item $ref (Join-Path $keep ($leaf + '.upfront')) -Force + Copy-Item $p (Join-Path $keep ($leaf + '.folded')) -Force + $m11Issues.Add((("{0} differs from the flag-up-front product OUTSIDE the " + + "known-unavailable views ({1}) - this is a NEW completeness loss, not " + + "the recorded one; both copies kept under {2}") -f + $leaf, ($m11Pass1Unavailable -join ', '), $keep)) + } + continue + } + $a = [IO.File]::ReadAllBytes($ref) + $b = [IO.File]::ReadAllBytes($p) + if ($a.Length -ne $b.Length -or + [Convert]::ToBase64String($a) -ne [Convert]::ToBase64String($b)) { + # Both copies are kept for diagnosis: "differs" is not actionable, and the + # run directory is deleted when the dataset finishes. + # Its PARENT, not $runRoot: the finally block drops $runRoot, which would + # delete the only evidence of the failure it just reported. + $keep = Join-Path (Split-Path $runRoot -Parent) ('mode11-diff-' + $name) + New-Item -ItemType Directory -Path $keep -Force | Out-Null + Copy-Item $ref (Join-Path $keep ($leaf + '.upfront')) -Force + Copy-Item $p (Join-Path $keep ($leaf + '.folded')) -Force + $m11Issues.Add((("{0} differs from the product the flag-up-front run wrote " + + "({1} vs {2} bytes) - the folded report is not the SAME report; both " + + "copies kept under {3}") -f $leaf, $a.Length, $b.Length, $keep)) + } + } + + # ORACLE 3: nothing but the report and its products moved. The pay-later path runs + # over a FINISHED analysis, so an artifact rewrite here corrupts the very run it + # was asked to describe - and would do it on the user's completed data. + $m11Allowed = @('output.1st-pass.model-diagnostics.json', + 'output.2nd-pass.model-diagnostics.json', + 'output.model-diagnostics.html') + # .osprey.task stamps are excluded, and that is not a loophole. A task that RUNS + # restamps every declared output it finds (AnalysisPipeline.WriteTaskSidecars), + # so the pass-1 fold necessarily re-stamps the first pass's per-file sidecars - + # that is the resume model recording that the task completed under this key, not + # the fold rewriting the analysis. The fingerprint keys on mtime, so an identical + # rewrite still shows. What must not move is the DATA, which is what remains + # asserted here; the stamps' correctness is mode 2's and mode 4's business. + $m11Changed = @(Compare-DirFingerprint -Before $m11Before -Dir $straightDir | + Where-Object { $_ -notmatch '\.log$' -and $_ -notmatch '\.osprey\.task$' }) + foreach ($c in $m11Changed) { + $leaf = ($c -replace '^[a-z]+: ', '') + # The products' own validity stamps travel with them. + $leaf = ($leaf -replace '\.(FirstPassFDR|SecondPassFDR)\.osprey\.task$', '') + if ($m11Allowed -notcontains $leaf) { + $m11Issues.Add(("the pay-later fold touched an artifact other than the " + + "report: {0}" -f $c)) + } + } + } + Remove-Item $m11Ref -Recurse -Force -ErrorAction SilentlyContinue + + if ($m11Issues.Count -eq 0) { + $summaryLines.Add(("$name mode11 (pay-later diagnostics: folded, no analysis, same " + + "report): PASS (pass-2 byte-exact; pass-1 exact except the views no pay-later " + + "path can rebuild today: {0})") -f ($m11Pass1Unavailable -join ', ')) + } else { + $overallFail = $true + Write-Problem-Tc "$name mode11 (pay-later diagnostics): FAIL - $($m11Issues.Count) issue(s)" + $m11Issues | Select-Object -First 15 | ForEach-Object { Write-Host " $_" -ForegroundColor Red } + $summaryLines.Add("$name mode11 (pay-later diagnostics): FAIL ($($m11Issues.Count) issues)") + } + } + # ---- mode 8: a PARTIALLY completed rescore resumes and FINISHES ---------------- # The state no other leg produces, which is why a real defect shipped. Mode 2 resumes from a # COMPLETE Stage-5 directory and mode 4 re-runs with EVERYTHING cached, so neither ever From 652043b0039cf259824f373ee2baa76a7a8a44ff Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 07:12:57 -0700 Subject: [PATCH 21/30] Declared the pass-2 diagnostics product so its fold can be reached * On --task SecondPassFDR the product was not a declared output, so a completed cohort had nothing outstanding, the driver skipped the task, and the pay-later fold could never run - measured as "skipping (outputs valid)" * The reason it was undeclared has expired: declaring it used to force a pass-2 re-join on every resume, and the fold arm makes it a bounded reduction instead * The predicate now excludes its own product, as pass 1's does; without that it reads its own missing product as an outstanding input and always declines * Verified at 446 files: exit 0 in 18 min at 14.0 GB managed / 22.1 GB private, below the 18.2/29.0 the same work costs inside the join, with the streamed marker and no analysis of any kind See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index ccd05e3b2e..204ea503ea 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -165,6 +165,27 @@ public override IEnumerable Outputs(PipelineContext ctx) if (ctx.Config.ModelDiagnostics && FirstPassFdrTask.IsIncludedFor(ctx.Config)) yield return ModelDiagnosticsReport.ReportPath(ctx.Config); + // The pass-2 diagnostics PRODUCT, declared whenever the flag is on and WITHOUT the + // IsIncludedFor term above - because on `--task SecondPassFDR` that term is false, + // so nothing this task owns was outstanding on a completed cohort, the driver + // skipped the task as already-done, and the pay-later fold could never run. Measured: + // a 10-file bed with every artifact current logged `SecondPassFDR: skipping (outputs + // valid)` and produced no pass-2 report at all. + // + // The reason it was NOT declared has expired, and that is what makes this safe now. + // The paragraph above records it: declaring a diagnostics output used to make the + // task permanently unskippable, because producing it meant re-running pass-2 + // Percolator, protein FDR and the whole blib write. With the fold arm this task now + // has, an outstanding pass-2 product costs a bounded reduction over artifacts that + // are already on disk - minutes, against the 69 minutes that join takes at 446 - so + // "outstanding" is no longer a reason to fear declaring it. + if (ctx.Config.ModelDiagnostics) + { + string pass2Product = ModelDiagnosticsReport.Pass2SidecarPath(ctx.Config); + if (!string.IsNullOrEmpty(pass2Product)) + yield return pass2Product; + } + // EVERY input file gets a 2nd-pass FDR sidecar, and they are declared here // unconditionally. This used to be gated on AnyReconciledParquet, so a run where // Stage 6 rescored nothing produced no 2nd-pass files at all - and a MISSING file @@ -571,8 +592,17 @@ private bool OnlyDiagnosticsProductOutstanding(PipelineContext ctx) string validityKey = ValidityKey(ctx); foreach (string output in Outputs(ctx)) { - if (string.Equals(output, reportPath, StringComparison.OrdinalIgnoreCase)) + // The page and the pass-2 product are what this arm EXISTS to write, so neither + // can be a reason to decline. A predicate asking "is every OTHER output current" + // has to exclude its own products - pass 1's version excludes its own for the + // same reason. Without the pass2Path term the arm reads its own missing product + // as an outstanding input and declines every time, which is a self-defeating + // condition that looks exactly like the fold not working. + if (string.Equals(output, reportPath, StringComparison.OrdinalIgnoreCase) || + string.Equals(output, pass2Path, StringComparison.OrdinalIgnoreCase)) + { continue; + } if (PerFileResumeDriver.IsCurrent(output, Name, validityKey)) continue; // Name the first output that failed. Declining here is not an error - it means From 22f16b246e9030b110d8b55e349ee564a0555ff9 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 10:18:26 -0700 Subject: [PATCH 22/30] Corrected the Stage-7 resident-gap disclosure the gate prints * It said "every leg of every dataset" while this gate's own mode-3 SecondPassFDR phase now takes the streamed join - the file already said so twelve lines below, so it contradicted itself and printed the stale half * The projection is now confirmed rather than projected: the model predicts 92.3 GB at 446 and 91.1 GB private was measured See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- pwiz_tools/Osprey/regression.ps1 | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 89e4547c81..944fa2277c 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -293,7 +293,16 @@ $knownResidentGaps = @( # the 4/8/16-file A/B. Quoting a straight-through 82-file endpoint next to that rig's # marginal slope produced three numbers no single model reproduced (24.43/82 = 0.298, # not 0.197), which is unreadable in a summary that prints on every CI run. - Legs = 'Every leg of every dataset. ~4.4 GB library + 0.197 GB/file live post-GC: ~20 GB at 82 files, ~103 GB projected at 500.' + # NOT every leg any more, and saying so mattered: this gate's own mode-3 SecondPassFDR + # phase takes the streamed join, which is the fix for this gap being exercised rather + # than merely described. The note below about hpc-merge already said that; this line + # still said "every leg", so the summary printed the stale half on every CI run. + # + # The model is now CONFIRMED rather than projected: 4.4 + 0.197*446 = 92.3 GB predicts + # the 91.1 GB private measured on the 446-run CHS cohort (2026-09-08), which is the + # first endpoint past 82 files. Quoted as a check on the model, not as a second model - + # see the note above about three numbers no single model reproduced. + Legs = 'Every leg EXCEPT the streamed Stage-7 join (mode 3''s SecondPassFDR phase, which sets ExpectReconciledInput). ~4.4 GB library + 0.197 GB/file live post-GC: ~20 GB at 82 files, ~103 GB projected at 500, and 92.3 GB predicted vs 91.1 GB measured at 446.' } ) # Reachable only outside this gate, tokened, each with an open issue: From 32ea5f6aa34e15505554f2198bf99c5bc778f9af Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 10:50:44 -0700 Subject: [PATCH 23/30] Tokened the Stage-7 resident join and gave the ordinary run a voice * OSPREY_STAGE7_STREAM=0 now requires OSPREY_ALLOW_UNFIXED_RESIDENT= stage7-stream-off, like the two sibling A/B oracles. It was untokened only because there was no alternative to choose; the streamed join made it a choice, and a choice is what the ratchet exists to record * The guard refuses ONLY the chosen case. A run that could not stream anyway is not refused - demanding a token there would make the default path unusable, which is the blanket amnesty the named-token scheme replaced * That case now WARNS instead, naming the shape, the issue and the cost model for its own file count. The deficiency was previously stated only in regression.ps1's summary, which prints for us and never for the operator whose run is about to take it * Guard placed first in Run: the diagnostics fold returns early and is itself a resident path when the join cannot stream, so a later placement silenced the warning on exactly the run that measured 91.1 GB at 446 files * streamingAvailable is a parameter so the refusal is unit-testable; computing it internally makes every test process answer false and pass vacuously See TODO-20260906_osprey_stage7_lean_row.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QnrzSdBPwztyJw1hfErF55 --- .../Osprey/Osprey.Core/ResidentPaths.cs | 30 ++++++++- .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 63 ++++++++++++++++++- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 38 +++++++++++ .../Osprey.Test/ResidentPoolGuardTest.cs | 55 +++++++++++++++- pwiz_tools/Osprey/regression.ps1 | 23 ++++--- 5 files changed, 196 insertions(+), 13 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs b/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs index 274ec84411..67d32da953 100644 --- a/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs +++ b/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs @@ -113,13 +113,41 @@ public static class ResidentPaths // the token had nothing left to admit. Not to be re-added - a resume that cannot stream // the Stage 6 handoff is a defect to fix, not a path to name. + /// + /// OSPREY_STAGE7_STREAM=0: the operator forced the RESIDENT Stage-7 join, where + /// SecondPassFDR rebuilds every run's survivors at once and holds them for the whole + /// stage - 4.4 GB library + 0.197 GB/file live post-GC, i.e. 91.1 GB measured on a + /// 446-run cohort. Like and + /// this is the A/B byte-identity oracle for a + /// streamed default, and it is named for exactly the same reason those two are. + /// + /// This entry ADDS to the list, which the class remarks say must only shrink, so + /// it owes the same justification gave. It names + /// a path that was previously UNNAMEABLE rather than re-admitting one that had been + /// fixed: until the streamed join existed there was no alternative, so a token could + /// only have been mandatory on every run, granting nothing. The alternative now exists, + /// which turns the resident arm from a fact into a CHOICE - and a choice is precisely + /// what this mechanism is for. Leaving it untokened is what let a green gate print + /// "Tokens REQUIRED: 0" while the fat path ran, which is why a second, parallel + /// disclosure table had to be invented to see it at all. + /// + /// It admits ONLY the chosen case. A run that takes the resident join because no + /// streamed one was admissible - a straight-through run, where + /// ExpectReconciledInput is false - is not refused by this token, because there + /// is nothing for the operator to choose. That case goes when + /// CanStreamStage7Join's admission stops being expressed as a CLI flag; then this + /// token is the only way to be resident, and it can be deleted with the switch. + /// + public static readonly string STAGE7_STREAM_OFF = @"stage7-stream-off"; + /// /// Every legal OSPREY_ALLOW_UNFIXED_RESIDENT value. Pinned by /// ResidentPoolGuardTest - see the class remarks for why it may only shrink. /// public static readonly IReadOnlyList KNOWN_UNFIXED = new[] { - FDRBENCH_PASS1, NON_PERCOLATOR_FDR, PROJECTION_OFF, COMPACTED_ENTRIES_BUFFER + FDRBENCH_PASS1, NON_PERCOLATOR_FDR, PROJECTION_OFF, COMPACTED_ENTRIES_BUFFER, + STAGE7_STREAM_OFF }; } } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index 61718b09ce..a0104ea85b 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -460,7 +460,18 @@ internal static bool CanHydratePerRun(OspreyConfig config) /// internal static bool CanStreamStage7Join(OspreyConfig config) { - if (!config.ExpectReconciledInput || !OspreyEnvironment.Stage7Stream) + return CanStreamStage7Join(config, OspreyEnvironment.Stage7Stream); + } + + /// + /// Pure core of the one-argument CanStreamStage7Join, with the env switch + /// passed in. Exists so can ask the question the + /// operator's choice hinges on - "would this run have streamed if the switch were on?" - + /// which is what separates a CHOSEN resident join from one that had no alternative. + /// + internal static bool CanStreamStage7Join(OspreyConfig config, bool stage7Stream) + { + if (!config.ExpectReconciledInput || !stage7Stream) return false; if (PerFileScoringTask.NeedsResidentPool(config, OspreyEnvironment.UseFdrProjection)) return false; @@ -494,13 +505,61 @@ internal static bool CanStreamStage7Join(OspreyConfig config) RetainedBaseIdSidecar.IsCurrentFormat(retainedPath); } + /// + /// Fail fast when the RESIDENT Stage-7 join was CHOSEN over an admissible streamed one, + /// unless the operator named . The Stage-7 + /// sibling of PerFileScoringTask.GuardResidentPool, which stops at the + /// pre-compaction line and so never saw this pool. + /// + /// Only the CHOSEN case. The question asked is "would this run have streamed with + /// the switch on", so a run that could not stream for any other reason - a + /// straight-through join, a non-protein-compact pass-2 mode, a missing retained-base_id + /// summary - is not refused, because there is no choice for a token to record. Those + /// remain disclosed rather than tokened until the streamed join is admissible for them + /// too; refusing them here would put a mandatory token on every ordinary run, which + /// grants nothing and is exactly the blanket amnesty the named-token ratchet replaced. + /// + /// streamingAvailable - whether this run COULD stream the join, i.e. + /// the two-argument CanStreamStage7Join with the switch forced on - is + /// passed IN rather than computed here, so the guard is a pure function and its refusal + /// is unit-testable. Computing it internally makes every test process answer false (no + /// retained-base_id sidecar on disk), so the refusal branch would never be reached and + /// the test would pass vacuously. Its Stage-6 sibling takes the same parameter for the + /// same reason. + /// + internal static string Stage7ResidentGuardError( + bool streamingAvailable, bool stage7Stream, string allowUnfixedResident) + { + if (stage7Stream || !streamingAvailable) + return null; + if (OspreyEnvironment.NamesResidentPath(allowUnfixedResident, + ResidentPaths.STAGE7_STREAM_OFF)) + { + return null; + } + // The SUPPLIED value is quoted, matching the two sibling guards: a stale or + // misspelled token otherwise reads exactly like an unset one, and the operator + // cannot tell "you named nothing" from "you named the wrong path". + return string.Format( + @"OSPREY_STAGE7_STREAM=0 forces the RESIDENT Stage-7 join, which rebuilds every " + + @"run's survivors at once and holds them for the whole stage - O(files), measured " + + @"at 91.1 GB on a 446-run cohort. This run CAN stream it, so residency here is a " + + @"choice and has to be named: set OSPREY_ALLOW_UNFIXED_RESIDENT={0} to run the " + + @"A/B deliberately, or unset OSPREY_STAGE7_STREAM to take the streamed join. " + + @"OSPREY_ALLOW_UNFIXED_RESIDENT is currently {1}.", + ResidentPaths.STAGE7_STREAM_OFF, + string.IsNullOrWhiteSpace(allowUnfixedResident) + ? @"unset" + : @"'" + allowUnfixedResident + @"'"); + } + /// /// The retained base_id set for the streamed second-pass join, or a hard failure. /// /// Separate from 's null-returning form because /// the CALLER cannot degrade here. By the time Stage 7 asks, the --input-scores /// load has already published one EMPTY list per run on the strength of - /// - which only header-probes the sidecar - so a null + /// CanStreamStage7Join - which only header-probes the sidecar - so a null /// leaves the stage folding over 446 empty runs, logging "No entries pass FDR threshold. /// Creating empty blib." and exiting 0. An empty .blib from a successful-looking /// run is the worst outcome this pipeline can produce, and the sidecar's own reader diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 204ea503ea..9e977ffc49 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -285,6 +285,39 @@ public override string ValidityKey(PipelineContext ctx) public override bool Run(PipelineContext ctx) { + bool couldStream = ScoringTaskShared.CanStreamStage7Join(ctx.Config, stage7Stream: true); + string residentError = ScoringTaskShared.Stage7ResidentGuardError( + couldStream, OspreyEnvironment.Stage7Stream, OspreyEnvironment.AllowUnfixedResident); + if (residentError != null) + throw new InvalidOperationException(residentError); + + // The run that CANNOT stream is not refused - there is no alternative to choose, and + // demanding a token would make the ordinary path unusable. But it must not be + // silent either. Until now the only statement of this deficiency lived in + // regression.ps1's end-of-run table, which prints for the developers who already + // know and never for the operator whose run is about to take it: what they get + // today is an OOM at a file count nothing warned them about. + // + // FIRST in Run, ahead of the diagnostics fold arm below. That arm returns early, and + // when the join cannot stream it is ITSELF a resident-pool path - it pulls the same + // survivor buffer, which is where 91.1 GB was measured at 446 files. Placing this + // after it silenced the warning on exactly the run that most needed it. + // + // One line, naming the shape, the issue and the cost model, so the ceiling is + // predictable from the run's own output rather than from a projection in a gate + // nobody outside this repo executes. + if (!couldStream) + { + ctx.LogWarning(string.Format( + @"Stage 7 is taking the RESIDENT join: every run's survivors are rebuilt at " + + @"once and held for the whole stage, which is O(files) (issue #4486). " + + @"Measured cost is ~4.4 GB plus ~0.197 GB per file, so {0} file(s) needs " + + @"~{1:F0} GB. The streamed join is admitted only for --task SecondPassFDR " + + @"today; this run does not qualify, so there is nothing to switch on.", + ctx.Config.InputFiles?.Count ?? 0, + 4.4 + 0.197 * (ctx.Config.InputFiles?.Count ?? 0))); + } + // The pass-2 diagnostics product is the ONLY outstanding output: every // computational artifact this task produces is already on disk and key-current, and // the driver reached Run solely because the pass-2 diagnostics JSON is missing - @@ -310,6 +343,11 @@ public override bool Run(PipelineContext ctx) return FoldPass2DiagnosticsOnly(ctx); } + // Refuse a resident Stage-7 join that was CHOSEN over an admissible streamed one, + // before anything is written or any pool is pulled. The first-pass guard cannot see + // this pool - it stops at the compaction line - so without this the fat path was + // reachable with no token at all, which is the one shape the named-token ratchet is + // supposed to make impossible. // Mid-Run crash safety: see FirstPassFdrTask.Run for rationale. foreach (var output in Outputs(ctx)) TaskValiditySidecar.Delete(output, Name); diff --git a/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs b/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs index db498c1f50..932da8e882 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs @@ -183,11 +183,16 @@ public void TestResidentPoolGuardError() // 'mdiag-full-resume' is GONE (#4505), 'resume-survivor-handoff' is GONE (#4536, // the rehydrate got its own survivor loader), and 'hpc-merge' is GONE (#4486, the // reconciled-input merge streams its load) - the ratchet shrinking three times. + // 'stage7-stream-off' was ADDED once the streamed Stage-7 join existed: until then + // the resident join had no alternative, so a token could only have been mandatory + // on every run and would have granted nothing. See the constant's own remarks for + // why naming a previously UNNAMEABLE path is the ratchet reaching further rather + // than running backwards. CollectionAssert.AreEqual( new[] { "fdrbench-pass1", "non-percolator-fdr", - "projection-off", "compacted-entries-buffer" + "projection-off", "compacted-entries-buffer", "stage7-stream-off" }, ResidentPaths.KNOWN_UNFIXED.ToArray()); @@ -197,6 +202,11 @@ public void TestResidentPoolGuardError() // refuse it. Streaming it is the default; the resident opt-out is a named path. AssertStage6HandoffGuard(); + // The STAGE-7 join guard. Same shape one stage later: the pre-compaction guard + // stops at the compaction line and the Stage-6 one at the handoff, so the survivor + // buffer SecondPassFDR rebuilds was refused by neither and no token could name it. + AssertStage7JoinGuard(); + // The trigger SET itself, not just the message it produces. Each of these takes the // O(files) resident pool and so arms the guard above. AssertNeedsResidentPool(true, fdrbench1); @@ -230,6 +240,49 @@ private static void AssertNeedsResidentPool(bool expected, OspreyConfig config) /// in the first place is not asked for a second token on top of the one its own /// resident path already requires. /// + /// + /// The Stage-7 join guard: a RESIDENT join CHOSEN over an admissible streamed one must + /// be named. Only the chosen case - a run that could not have streamed anyway is not + /// refused, because there is nothing for a token to record and demanding one would put + /// a mandatory token on every ordinary run. + /// + private static void AssertStage7JoinGuard() + { + // Streaming on: no error, whatever the token says. + Assert.IsNull(ScoringTaskShared.Stage7ResidentGuardError( + streamingAvailable: true, stage7Stream: true, allowUnfixedResident: null)); + Assert.IsNull(ScoringTaskShared.Stage7ResidentGuardError( + true, true, ResidentPaths.FDRBENCH_PASS1)); + + // OSPREY_STAGE7_STREAM=0 on a run that COULD stream: refused, and the message names + // the token to set rather than describing a symptom. + string err = ScoringTaskShared.Stage7ResidentGuardError(true, false, null); + Assert.IsNotNull(err); + StringAssert.Contains(err, + "OSPREY_ALLOW_UNFIXED_RESIDENT=" + ResidentPaths.STAGE7_STREAM_OFF); + + // Naming THIS path admits it - that is the A/B byte-identity oracle. Case- + // insensitive, matching both sibling guards. + Assert.IsNull(ScoringTaskShared.Stage7ResidentGuardError( + true, false, ResidentPaths.STAGE7_STREAM_OFF)); + Assert.IsNull(ScoringTaskShared.Stage7ResidentGuardError( + true, false, ResidentPaths.STAGE7_STREAM_OFF.ToUpperInvariant())); + + // Naming a DIFFERENT path does not, and the message quotes the value supplied so a + // stale token does not read like an unset one. + string wrongToken = ScoringTaskShared.Stage7ResidentGuardError( + true, false, ResidentPaths.PROJECTION_OFF); + Assert.IsNotNull(wrongToken); + StringAssert.Contains(wrongToken, ResidentPaths.PROJECTION_OFF); + + // A run that could not stream ANYWAY is NOT guarded here - there is no choice for a + // token to record. This is the straight-through join, and it is precisely the case + // that keeps the DEFAULT path usable without demanding a token: refusing it would + // put a mandatory token on every ordinary run, which grants nothing. + Assert.IsNull(ScoringTaskShared.Stage7ResidentGuardError( + streamingAvailable: false, stage7Stream: false, allowUnfixedResident: null)); + } + private static void AssertStage6HandoffGuard() { // Streaming: no error, whatever the token says. diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 944fa2277c..6c1dd1b00f 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -322,16 +322,21 @@ $script:priorAllowResident = $env:OSPREY_ALLOW_UNFIXED_RESIDENT # comparison this harness exists to support impossible to run. Ambient tokens are stripped # ONLY when no such switch is set, which is the case the clearing is aimed at. # -# OSPREY_STAGE7_STREAM=0 is deliberately NOT in this set, though it also forces a resident -# path. The list is not "switches that select a resident path", it is "switches that arm a -# guard which REFUSES without a token" - ResidentPaths.KNOWN_UNFIXED is exactly -# { FDRBENCH_PASS1, NON_PERCOLATOR_FDR, PROJECTION_OFF, COMPACTED_ENTRIES_BUFFER } and has -# no Stage-7 entry, which is also why $knownResidentGaps records the Stage-7 pool with -# Token = 'NONE'. Adding it would keep an ambient OSPREY_ALLOW_UNFIXED_RESIDENT alive across -# every leg of every dataset in exchange for nothing, which is the named-token ratchet the -# preamble above exists to enforce, weakened. +# OSPREY_STAGE7_STREAM=0 IS in this set now. It was excluded while the reasoning was +# circular - "the list is switches that arm a guard which refuses without a token, and +# KNOWN_UNFIXED has no Stage-7 entry" says only that there was no token because there was no +# token. There was no token because there was no ALTERNATIVE: until the streamed Stage-7 join +# existed the resident one was a fact, and a token can only be demanded for a choice. The +# alternative exists, so the switch is now exactly what OSPREY_FDR_PROJECTION=0 and +# OSPREY_STAGE6_STREAM_SURVIVORS=0 already were - a deliberate A/B oracle forcing a fat path - +# and it is tokened like them (ResidentPaths.STAGE7_STREAM_OFF). +# +# This costs the gate nothing: no leg sets OSPREY_STAGE7_STREAM, so no leg needs the token and +# the required-token count below stays 0. It exists so an OPERATOR running the A/B is not +# aborted on the first leg, which is what this whole block is for. $abSwitchSet = ($env:OSPREY_STAGE6_STREAM_SURVIVORS -eq '0') -or - ($env:OSPREY_FDR_PROJECTION -eq '0') + ($env:OSPREY_FDR_PROJECTION -eq '0') -or + ($env:OSPREY_STAGE7_STREAM -eq '0') if (-not [string]::IsNullOrWhiteSpace($env:OSPREY_ALLOW_UNFIXED_RESIDENT)) { if ($abSwitchSet) { # Extra parens: -f binds TIGHTER than +, so without them only the LAST fragment is From 40997e2ca1add257eae21d0cc945a9813392777e Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 13:53:09 -0700 Subject: [PATCH 24/30] Made the straight-through Stage-7 join fold run by run * Derived CanStreamStage7Join from the reconciled parquets on disk, retiring the --task SecondPassFDR proxy that made the ordinary run resident * Gave both straight-through arms a per-run source, built from the whole-run loops' own per-file halves so the arms cannot drift * Keyed the O(files) warning on the milestone rather than the predicate, and asserted the per-run fold per leg in regression.ps1 See TODO-20260908_osprey_stage7_straightthrough_stream.md in pwiz-ai/todos Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QwgkvKC6UwFUNieMDvpw57 --- .../Osprey/Osprey.Core/ResidentPaths.cs | 14 +- .../Osprey/Osprey.IO/ParquetScoreCache.cs | 29 +++ .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 246 ++++++++++++++++-- .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 80 +++++- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 89 ++++--- pwiz_tools/Osprey/Osprey.Test/IOTest.cs | 60 +++++ .../Osprey.Test/ResidentPoolGuardTest.cs | 43 +++ .../Osprey/docs/00-pipeline-architecture.md | 35 ++- pwiz_tools/Osprey/regression.ps1 | 99 ++++++- 9 files changed, 602 insertions(+), 93 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs b/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs index 67d32da953..494204fdc0 100644 --- a/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs +++ b/pwiz_tools/Osprey/Osprey.Core/ResidentPaths.cs @@ -132,11 +132,15 @@ public static class ResidentPaths /// disclosure table had to be invented to see it at all. /// /// It admits ONLY the chosen case. A run that takes the resident join because no - /// streamed one was admissible - a straight-through run, where - /// ExpectReconciledInput is false - is not refused by this token, because there - /// is nothing for the operator to choose. That case goes when - /// CanStreamStage7Join's admission stops being expressed as a CLI flag; then this - /// token is the only way to be resident, and it can be deleted with the switch. + /// streamed one was admissible is not refused by this token, because there is nothing + /// for the operator to choose. That set has SHRUNK to one: the straight-through run + /// used to be in it - CanStreamStage7Join's admission was the CLI flag + /// ExpectReconciledInput, so the ordinary run could not stream by construction - + /// and it is now derived from the reconciled parquets on disk, which every route + /// satisfies. What is left is a pass-2 mode whose per-file half has no worker + /// (OSPREY_PASS2_QVALUE=transfer), and when that half moves to + /// Pass2PerFileWorker this exemption has no subject, the guard can refuse + /// unconditionally, and this token becomes the only way to be resident. /// public static readonly string STAGE7_STREAM_OFF = @"stage7-stream-off"; diff --git a/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs b/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs index 5140547189..c9ed15cf77 100644 --- a/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs +++ b/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs @@ -255,6 +255,35 @@ public static bool IsSubsetWithoutScoreIndex(string path) return !HasColumn(path, FIELD_SCORE_INDEX.Name); } + /// + /// True when is a reconciled parquet THIS build can read in the + /// survivor-subset shape: it exists, carries the current + /// marker, and carries the score_index column + /// that ties each survivor row back to its Stage 4 ordinal. + /// + /// The positive form of plus the marker + /// test, in one open, because two callers ask the same question about the same file and + /// asking it twice is what let them drift. One is the Stage 7 refusal that names the + /// stale files; the other is the admission a per-run fold consults BEFORE it commits to + /// rebuilding each run from these parquets - and that admission has to be the SAME + /// question the refusal asks, or a run is admitted to a fold it then aborts. + /// + /// Says nothing about whether Stage 6 did any rescore WORK on the file - that is + /// the osprey.rescored footer key and a different question, deciding whether a + /// second Percolator pass is owed. This one asks only whether the rows are readable in + /// the shape a survivor rebuild needs. + /// + public static bool IsCurrentReconciledSurvivorSubset(string path) + { + if (string.IsNullOrEmpty(path) || !File.Exists(path)) + return false; + var footer = LoadFooterMetadata(path); + footer.TryGetValue(@"osprey.reconciled", out string marker); + if (!string.Equals(marker, RECONCILED_SURVIVORS, StringComparison.Ordinal)) + return false; + return HasColumn(path, FIELD_SCORE_INDEX.Name); + } + /// Whether a parquet's schema carries a column by this name. public static bool HasColumn(string path, string columnName) { diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index 2999ba8ef5..8fdc2d62db 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -318,22 +318,19 @@ public override bool Run(PipelineContext ctx) // it dropped, so streaming there would destroy the only copy of the survivors on // its way past. Null leaves StreamFiles walking the resident buffer, which is what // the OSPREY_STAGE6_STREAM_SURVIVORS=0 A/B oracle needs it to do. - // NO per-file source on this path, and the reason is a property of the materializer - // rather than a preference. MaterializeRescoredFile is ONE-SHOT: it overlays the - // reconciled parquet and appends gap-fill rows, so calling it twice for one run - // duplicates them - which is the same "run-once, and a failed build stays failed" - // rule RescoredEntries' own remarks state for the whole-run build. A Stage 7 fold - // re-enumerates, so it needs a source that can rebuild a run from disk repeatedly - // and identically; this one cannot, and handing it over produced a straight-through - // Stellar run that exited 1 on AssertSidecarDescribesPool. - // - // The leg that has such a source is the reconciled-input merge, where - // BuildStage7PerRunSource supplies it (Rehydrate, below). Straight-through Stage 7 - // therefore keeps its resident pool for now. That is the honest state and not a - // hidden one: making this materializer idempotent - or having it rebuild from the - // reconciled parquet alone, which already holds the merged gap-fill rows - is what - // extends the fold to this leg, and it is separate work. - var rescored = new RescoredEntries(_perFileEntries, () => BuildRescoredPool(ctx)); + // WITH a per-file source when this run can supply a re-enumerable one, which is + // what makes the bounded join the DEFAULT rather than a property of + // --task SecondPassFDR. The paragraph that stood here said this leg could not have + // one because MaterializeRescoredFile is ONE-SHOT - it overlays the reconciled + // parquet and appends gap-fill rows, so a second call for one run duplicates them, + // which once exited a straight-through Stellar run 1 on AssertSidecarDescribesPool. + // True, and true of a NON-EMPTY list only: given an empty one it takes its + // rebuild-from-the-reconciled-parquet branch, where the gap-fill rows are already + // merged and no overlay runs, and that branch repeats identically. So the condition + // is not "make the materializer idempotent" but "hand it a list it can rebuild", + // which is what BuildRunPerRunSource establishes before offering the source at all. + var rescored = new RescoredEntries(_perFileEntries, () => BuildRescoredPool(ctx), + BuildRunPerRunSource(ctx, survivorLoader)); ctx.Publish(rescored); // Self-gate: rescore + reconciliation only run when there is @@ -654,6 +651,30 @@ public override bool Rehydrate(PipelineContext ctx) { _perFileEntries = ctx.Get().Value; + // The SAME conversion the reconciled-input arm below already had, on the arm + // that carries every ordinary run. Both arms end with a milestone over the same + // shared buffer; what the source changes is that Stage 7 rebuilds one run, folds + // it and drops it, instead of the two whole-run loops beneath this block + // bringing all 446 to their post-rescore state and holding them - 91.1 GB + // private, measured on a 446-run resume, which is what admitting only + // --task SecondPassFDR to the fold left standing. + // + // Null when this run has no way to rebuild a dropped run, and then the resident + // loops below run exactly as they did. LAZY, unlike those loops: with a source + // in hand there is no reason to do the work before the consumer that folds asks + // for it, and a consumer that reads .Value instead still gets the whole pool + // through the same source (MaterializeAllFromSource), reported as the expense + // it is. + var resumeSource = BuildResumePerRunSource(ctx); + if (resumeSource != null) + { + var resumeBuffer = _perFileEntries; + ctx.Publish(new RescoredEntries(resumeBuffer, + () => MaterializeAllFromSource(resumeBuffer, resumeSource, ctx), + resumeSource)); + return true; + } + // PR-E: a fresh ExecuteRescore would overlay each file's reconciled // boundaries/area/features onto its CompactedEntries rows + append // gap-fill. On resume the driver skipped Run because the reconciled @@ -2082,6 +2103,67 @@ private static void MaterializeAllFromSource( } } + /// + /// The per-run source Stage 7 folds through on the STRAIGHT-THROUGH resume: bring ONE + /// run's list to the post-rescore state the two whole-run loops in + /// would have brought every run to, so the join holds one run at a time. + /// + /// Its body is those loops' own per-file halves, called in the same order rather + /// than reimplemented: refill the released survivors, then overlay this run's reconciled + /// parquet. Nothing in either half reads another run's entries, so run-at-a-time is the + /// same work in the same order as all-runs-then-all-runs - which is why the streamed and + /// resident arms write byte-identical output, and why the fix is a call-shape change + /// rather than a second implementation. + /// + /// RE-ENUMERABLE, which is the property + /// needs: a fold pass drops each run's list, so a second pass finds it empty, and both + /// halves here rebuild an empty list from disk identically. Unconditionally so, unlike + /// : a resume runs no rescore, so no run's list ever + /// holds state that is not already on disk, and there is nothing a drop can lose. + /// + /// Null when the run cannot stream. Two ways: the admission itself + /// (, which is where + /// the reconciled parquets are required to be readable - asked in FULL here, unlike + /// , because a resume enters after Stage 6 has + /// written them), or no published survivor loader - and the second is not a preference. + /// A run whose survivors were never released has nothing on disk to rebuild them from, + /// so folding would DROP the only copy on its way past. That is the same condition + /// states for its own deferred build; the difference is only that + /// this arm asks for the loader without the Stage-6 switch (see + /// ). A null loader today means an empty join, + /// where there is no pool to bound and the resident loops cost nothing. + /// + private Action> BuildResumePerRunSource(PipelineContext ctx) + { + if (!ScoringTaskShared.CanStreamStage7Join(ctx.Config)) + return null; + var loader = PublishedSurvivorLoader(ctx); + if (loader == null) + return null; + // Both answers taken ONCE, here, for the reason RescoredPoolPlan gives: the + // reconciled-parquet judgement stops being true the moment this task returns, when + // the driver stamps a fresh validity sidecar onto every declared output that merely + // exists. The answer travels; the question does not. + var reconciledPaths = CurrentReconciledPaths(ctx); + var gapFill = ctx.Get().Value; + // Says which shape Stage 7 took, for the reason its --task SecondPassFDR sibling + // gives: without it the only evidence is a memory profile, and "the gate is green so + // the new path must have run" is the inference that lets a resident path pass as a + // streamed one. + ctx.LogInfo(string.Format( + @"Second-pass join: folding over {0} run(s), each rebuilt from its own first-pass " + + @"survivors and dropped (no all-runs survivor pool). {1} of {0} run(s) carry a " + + @"current reconciled parquet to overlay; the rest keep their 1st-pass boundaries, " + + @"as they would on a fresh run.", + _perFileEntries.Count, reconciledPaths.Count)); + return (fileName, survivors) => + { + MaterializeFileSurvivors(fileName, survivors, loader, ctx); + OverlayReconciledIntoFile(fileName, survivors, reconciledPaths, gapFill, + canonicalize: true); + }; + } + private static Action> BuildStage7PerRunSource( IReadOnlyDictionary perFileParquetPaths, OspreyConfig config, @@ -2531,6 +2613,23 @@ private static FirstPassSurvivorLoader StreamedSurvivorLoader(PipelineContext ct { if (!OspreyEnvironment.Stage6StreamSurvivors) return null; + return PublishedSurvivorLoader(ctx); + } + + /// + /// The per-run survivor loader FirstPassFDR published, WITHOUT the Stage-6 switch + /// applies. + /// + /// One switch per stage. OSPREY_STAGE6_STREAM_SURVIVORS=0 is the A/B oracle + /// for the RESCORE window - it asks Stage 6 to keep the buffer it would have drained - + /// and it says nothing about how Stage 7 should fold. Reading it through the Stage-6 + /// gate would have made that oracle silently decide the Stage-7 arm as well, so the + /// stage whose arm OSPREY_STAGE7_STREAM governs asks for the loader directly. + /// The object is the same one either way - the gate withholds it, it does not + /// unbuild it. + /// + private static FirstPassSurvivorLoader PublishedSurvivorLoader(PipelineContext ctx) + { return ctx.TryGet(out var source) ? source?.Value : null; } @@ -2618,17 +2717,120 @@ public static RescoredPoolPlan RefillOnly( /// and lands inside no other stage's, so without one a perf comparison reads a 16-minute /// Stage 6 saving with nothing anywhere absorbing it. /// - private void BuildRescoredPool(PipelineContext ctx) + /// + /// The per-run source Stage 7 folds through on the straight-through COMPUTE path, or + /// null when this run cannot supply one and the whole-run + /// stays the only route. + /// + /// Its body is - the per-file half the + /// whole-run build already loops over - so run-at-a-time is the same work in the same + /// order, which is why the two arms write byte-identical output. + /// + /// A third condition cannot be asked here, and is enforced at fold time instead: + /// a run whose reconciled parquet never reached disk keeps its re-scored entries in + /// memory, and a fold that drops them loses the only copy. Only the rescore this + /// method runs BEFORE can decide that, so the source raises it when asked. + /// + /// Two conditions here, and neither is a preference. A run with no survivor loader kept + /// its resident buffer and has nothing on disk to rebuild a run from, so folding would + /// DROP the only copy on its way past. And every list must ALREADY be empty: that is + /// what makes repeatable, because an empty list + /// takes its rebuild-from-the-reconciled-parquet branch and skips the gap-fill-appending + /// overlay that a second call would apply twice. Both hold together exactly when + /// FirstPassFDR released the survivors, which is the default; the checked form is used + /// rather than that inference because the inference is the kind that stops being true + /// quietly. + /// + /// Asked WITHOUT the reconciled-parquet term + /// (): this decision is + /// taken at the top of Stage 6, and the parquets the full predicate asks about are what + /// the rescore below is about to write. This arm does not need them - a run without a + /// reconciled sibling is rebuilt from its Stage 4 parquet plus its 1st-pass sidecar, + /// which is the no-work file's normal path here. + /// + private Action> BuildRunPerRunSource( + PipelineContext ctx, FirstPassSurvivorLoader survivorLoader) + { + if (survivorLoader == null || + !ScoringTaskShared.Stage7StreamAdmittedBeforeRescore( + ctx.Config, OspreyEnvironment.Stage7Stream)) + { + return null; + } + foreach (var kv in _perFileEntries) + { + if (kv.Value.Count > 0) + return null; + } + // The SAME opening words as the other two arms' markers, deliberately: the gate + // asserts the per-run fold by that phrase, and an arm that streams under a different + // sentence is an arm no verifier can see. Said HERE, at the top of Stage 6, because + // this is where the decision is taken - a worker that exits before Stage 7 has still + // made it, and the alternative (report it when the fold starts) is a fact about the + // consumer rather than about this task. + ctx.LogInfo(string.Format( + @"Second-pass join: folding over {0} run(s), each rebuilt from its own artifacts " + + @"and dropped (no all-runs survivor pool). Decided at Stage 6, where the " + + @"survivors were released.", _perFileEntries.Count)); + return (fileName, entries) => + { + var plan = PoolPlanForBuild(); + // THE ONE RUN THE FOLD CANNOT SERVE, and it fails rather than folding it. + // ExecuteRescore drops a run's entries only when its reconciled parquet reached + // disk, KEEPING them when the write no-opped or failed - because in that case + // those entries are the only copy of the rescore. A fold drops every run it + // hands over, so streaming such a run would discard that copy and the next pass + // would rebuild it from the Stage 4 parquet: a blib silently carrying 1st-pass + // boundaries for one run, from a run that exits 0. The resident build survives + // it by never dropping anything, which is why this is new here rather than a + // defect being uncovered. + // + // RescoredFiles null means no rescore ran at all (the self-gated refill-only + // plan), and then no run has - or needs - a reconciled parquet: every one is + // rebuilt from its Stage 4 parquet plus its 1st-pass sidecar, repeatably. It is + // only a run that WAS rescored and has no current reconciled parquet that has + // state nothing on disk holds. + if (plan.RescoredFiles != null && + (plan.ReconciledPaths == null || !plan.ReconciledPaths.ContainsKey(fileName))) + { + throw new InvalidDataException(string.Format( + @"Second-pass join: run '{0}' has no current .scores-reconciled.parquet, " + + @"so its re-scored survivors exist only in memory and the per-run fold " + + @"cannot rebuild them. Stage 6 did not persist this run - it logged a " + + @"warning when the write no-opped or failed. Re-run Stage 6 for it.", + fileName)); + } + // CLEARED here rather than relying on the caller having dropped the run. + // StreamFiles does drop it, but MaterializeFile leaves that to its caller, and + // the whole repeatability argument above rests on the list being empty - so the + // source establishes that itself instead of inheriting it from a call site. + entries.Clear(); + // plan.Loader is non-null by construction: both plan branches are built from + // the same survivorLoader this method already refused to proceed without. + MaterializeRescoredFile(ctx, plan, fileName, entries); + }; + } + + /// + /// The plan parked, or a hard failure. + /// + /// A pull before Run decided is a programming defect, not a case to guess at: + /// every guess available here - refill-only, or an overlay against paths not yet judged - + /// silently produces a wrong reported set rather than an error. + /// + private RescoredPoolPlan PoolPlanForBuild() { - // A pull before Run decided is a programming defect, not a case to guess at: every - // guess available here (refill-only, or an overlay against paths not yet judged) - // silently produces a wrong reported set rather than an error. - var plan = _poolPlan; - if (plan == null) + if (_poolPlan == null) { throw new InvalidOperationException( @"RescoredEntries was pulled before PerFileRescoring decided how to build the survivor pool."); } + return _poolPlan; + } + + private void BuildRescoredPool(PipelineContext ctx) + { + var plan = PoolPlanForBuild(); if (plan.Loader == null) return; var sw = Stopwatch.StartNew(); diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index a0104ea85b..139e4ebcf8 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -447,9 +447,18 @@ internal static bool CanHydratePerRun(OspreyConfig config) /// that leg. Widening the first would have told the rescore it may stream on a leg where /// it does not run at all. /// - /// Three requirements, and the third is the one that is easy to miss. The leg has - /// to be the reconciled-input merge, whose parquets already hold the survivor subset. - /// No consumer may read PIN features off these stubs + /// Three requirements, and the first is the one that is easy to miss. Every run's + /// .scores-reconciled.parquet has to be on disk in the survivor-subset shape, + /// because that parquet IS what a run is rebuilt from and dropped again. It used to be + /// asked as config.ExpectReconciledInput, a PROXY for it: only + /// --task SecondPassFDR sets that flag, so the one route the #4486 repro used + /// became the only route that could stream - while a straight-through run, whose + /// Stage 6 had just written those same parquets, met the requirement and was refused + /// anyway (91.1 GB private, measured on a 446-run resume). Asked of the disk it is + /// route-independent, which is also what lets --input-scores retire without + /// taking the streamed join with it. + /// + /// No consumer may read PIN features off these stubs /// (PerFileScoringTask.NeedsResidentPool: --fdrbench-pass 1, a /// non-Percolator FDR method, OSPREY_FDR_PROJECTION=0) - a streamed pool drops /// the entries those consumers index. And the analysis-wide retained base_id summary has @@ -471,7 +480,34 @@ internal static bool CanStreamStage7Join(OspreyConfig config) /// internal static bool CanStreamStage7Join(OspreyConfig config, bool stage7Stream) { - if (!config.ExpectReconciledInput || !stage7Stream) + // LAST, because AllReconciledParquetsCurrent is the only term that opens a file per + // run. Every cheaper disqualifier returns first, so a run that was never going to + // stream does not pay 446 footer reads to be told so. + return Stage7StreamAdmittedBeforeRescore(config, stage7Stream) && + AllReconciledParquetsCurrent(config); + } + + /// + /// Every CanStreamStage7Join term EXCEPT the reconciled parquets, i.e. the half a + /// run can answer BEFORE Stage 6 has written them. + /// + /// Split out for exactly one caller: the straight-through Run arm of + /// PerFileRescoreTask, which decides whether to publish a per-run source at the + /// TOP of Stage 6, hours before the rescore it is about to perform writes the parquets + /// the full predicate asks about. Asking the full question there answers "no" on every + /// cold run - not because the run cannot stream, but because it has not got there yet. + /// It does not need the term either: that arm rebuilds a run through + /// , which falls back to the Stage 4 parquet plus + /// the 1st-pass sidecar for a run with no reconciled sibling, where the reconciled-input + /// merge has nothing else to read. + /// + /// The retained base_id summary IS in this half even though it is an artifact: + /// FirstPassFDR writes it before any caller of either form runs, so it is answerable on + /// every route at every point either question is asked. + /// + internal static bool Stage7StreamAdmittedBeforeRescore(OspreyConfig config, bool stage7Stream) + { + if (!stage7Stream) return false; if (PerFileScoringTask.NeedsResidentPool(config, OspreyEnvironment.UseFdrProjection)) return false; @@ -505,6 +541,42 @@ internal static bool CanStreamStage7Join(OspreyConfig config, bool stage7Stream) RetainedBaseIdSidecar.IsCurrentFormat(retainedPath); } + /// + /// True when EVERY input has a .scores-reconciled.parquet on disk that this build + /// can read in the survivor-subset shape - the disk-side question + /// config.ExpectReconciledInput used to stand in for. + /// + /// ALL, not any. The fold rebuilds each run from its own reconciled parquet, so + /// one run without a readable one is a run the fold cannot produce - and admitting the + /// stage on "some run has one" would fail at that run, hours in. The sibling question + /// "did Stage 6 rescore anything", which decides whether a second Percolator pass is + /// owed, is SecondPassFdrTask.AnyReconciledParquet and is deliberately not this: + /// a file with no rescore work still gets a faithful copy written for it, which is what + /// makes "all present" reachable on any route. + /// + /// Empty or absent inputs return FALSE rather than vacuously true. There is no + /// pool to bound on a run with no inputs, so the streamed arm buys nothing there, and + /// vacuous truth would hand the fold an empty file set on a configuration nothing else + /// in this predicate examines. + /// + internal static bool AllReconciledParquetsCurrent(OspreyConfig config) + { + if (config.InputFiles == null || config.InputFiles.Count == 0) + return false; + foreach (var input in config.InputFiles) + { + // From the INPUT stem, the same derivation AnyReconciledParquet uses, so this + // reads identically in the in-process pipeline (Stage 6 has just written the + // parquets) and on a --task SecondPassFDR node (a Stage 6 worker wrote them). + if (!ParquetScoreCache.IsCurrentReconciledSurvivorSubset( + ParquetScoreCache.GetReconciledScoresPath(input))) + { + return false; + } + } + return true; + } + /// /// Fail fast when the RESIDENT Stage-7 join was CHOSEN over an admissible streamed one, /// unless the operator named . The Stage-7 diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 9e977ffc49..f9d9534539 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -285,39 +285,18 @@ public override string ValidityKey(PipelineContext ctx) public override bool Run(PipelineContext ctx) { + // Refuse a resident Stage-7 join that was CHOSEN over an admissible streamed one, + // before anything is written or any pool is pulled. The first-pass guard cannot see + // this pool - it stops at the compaction line - so without this the fat path was + // reachable with no token at all, which is the one shape the named-token ratchet is + // supposed to make impossible. (The paragraph sat above the marker wipe below, far + // from the call it describes.) bool couldStream = ScoringTaskShared.CanStreamStage7Join(ctx.Config, stage7Stream: true); string residentError = ScoringTaskShared.Stage7ResidentGuardError( couldStream, OspreyEnvironment.Stage7Stream, OspreyEnvironment.AllowUnfixedResident); if (residentError != null) throw new InvalidOperationException(residentError); - // The run that CANNOT stream is not refused - there is no alternative to choose, and - // demanding a token would make the ordinary path unusable. But it must not be - // silent either. Until now the only statement of this deficiency lived in - // regression.ps1's end-of-run table, which prints for the developers who already - // know and never for the operator whose run is about to take it: what they get - // today is an OOM at a file count nothing warned them about. - // - // FIRST in Run, ahead of the diagnostics fold arm below. That arm returns early, and - // when the join cannot stream it is ITSELF a resident-pool path - it pulls the same - // survivor buffer, which is where 91.1 GB was measured at 446 files. Placing this - // after it silenced the warning on exactly the run that most needed it. - // - // One line, naming the shape, the issue and the cost model, so the ceiling is - // predictable from the run's own output rather than from a projection in a gate - // nobody outside this repo executes. - if (!couldStream) - { - ctx.LogWarning(string.Format( - @"Stage 7 is taking the RESIDENT join: every run's survivors are rebuilt at " + - @"once and held for the whole stage, which is O(files) (issue #4486). " + - @"Measured cost is ~4.4 GB plus ~0.197 GB per file, so {0} file(s) needs " + - @"~{1:F0} GB. The streamed join is admitted only for --task SecondPassFDR " + - @"today; this run does not qualify, so there is nothing to switch on.", - ctx.Config.InputFiles?.Count ?? 0, - 4.4 + 0.197 * (ctx.Config.InputFiles?.Count ?? 0))); - } - // The pass-2 diagnostics product is the ONLY outstanding output: every // computational artifact this task produces is already on disk and key-current, and // the driver reached Run solely because the pass-2 diagnostics JSON is missing - @@ -343,11 +322,6 @@ public override bool Run(PipelineContext ctx) return FoldPass2DiagnosticsOnly(ctx); } - // Refuse a resident Stage-7 join that was CHOSEN over an admissible streamed one, - // before anything is written or any pool is pulled. The first-pass guard cannot see - // this pool - it stops at the compaction line - so without this the fat path was - // reachable with no token at all, which is the one shape the named-token ratchet is - // supposed to make impossible. // Mid-Run crash safety: see FirstPassFdrTask.Run for rationale. foreach (var output in Outputs(ctx)) TaskValiditySidecar.Delete(output, Name); @@ -365,6 +339,7 @@ public override bool Run(PipelineContext ctx) // the work lands, and a worker that never reaches this line never pays it. // Taken as a TOKEN first, so the probes below can bracket that build. var rescored = ctx.Get(); + WarnResidentStage7Join(rescored, ctx); // Stage 7's INHERITED baseline, post-GC, before this stage does any work // (#4486). Every figure that issue has ever quoted came from --memstamp, i.e. @@ -657,6 +632,39 @@ private bool OnlyDiagnosticsProductOutstanding(PipelineContext ctx) return true; } + /// + /// Say so when Stage 7 is about to build the whole-run survivor pool, naming the shape, + /// the issue and the cost model. + /// + /// The run that CANNOT stream is not refused - there is no alternative to choose, + /// and demanding a token would make the ordinary path unusable. But it must not be + /// silent either: until #4642 the only statement of this deficiency lived in + /// regression.ps1's end-of-run table, which prints for the developers who already know + /// and never for the operator whose run is about to take it, and what they got instead + /// was an OOM at a file count nothing had warned them about. + /// + /// Keyed on the MILESTONE, not on CanStreamStage7Join. The predicate says + /// the run is ADMISSIBLE; only the milestone says a per-run source was actually built, + /// and the two stopped being the same statement once the admission was derived from disk + /// rather than named by one CLI flag. Reading the fact off the thing that decides it is + /// what keeps this line honest as each remaining arm is converted. + /// + /// Called from both arms that pull the milestone, each right after its pull. The + /// diagnostics-only fold returns before reaches its own call, so a + /// single site would go silent on exactly the run that most needs it. + /// + private static void WarnResidentStage7Join(RescoredEntries rescored, PipelineContext ctx) + { + if (rescored.Streams) + return; + int nFiles = ctx.Config.InputFiles?.Count ?? 0; + ctx.LogWarning(string.Format( + @"Stage 7 is taking the RESIDENT join: every run's survivors are rebuilt at " + + @"once and held for the whole stage, which is O(files) (issue #4486). " + + @"Measured cost is ~4.4 GB plus ~0.197 GB per file, so {0} file(s) needs " + + @"~{1:F0} GB.", nFiles, 4.4 + 0.197 * nFiles)); + } + /// /// Produce the pass-2 diagnostics product and nothing else, from a second pass that is /// already complete on disk. The pass-2 sibling of @@ -686,6 +694,10 @@ private bool FoldPass2DiagnosticsOnly(PipelineContext ctx) { var config = ctx.Config; var rescored = ctx.Get(); + // This arm is ITSELF a resident-pool path when the source is absent - it pulls the + // same survivor buffer, which is where 91.1 GB was measured at 446 files - and it + // returns before Run's own call, so it has to make the statement itself. + WarnResidentStage7Join(rescored, ctx); var libraryById = ctx.Get().Value; var perFileParquetPaths = ctx.Get().Value; @@ -1089,8 +1101,6 @@ private static List StaleReconciledParquets( string reconciledPath = ParquetScoreCache.ReconciledPathFromScoresPath(scoresPath); if (!File.Exists(reconciledPath)) continue; - var metadata = ParquetScoreCache.LoadFooterMetadata(reconciledPath); - metadata.TryGetValue(@"osprey.reconciled", out string marker); // Stale is EITHER an older generation (marker mismatch) OR the interim // #4486 shape - survivor subset with no score_index column - which the // per-file loaders would otherwise read by POSITION, silently binding @@ -1098,12 +1108,13 @@ private static List StaleReconciledParquets( // IsSubsetWithoutScoreIndex documents). Only FirstPassSurvivorLoader // carried that refusal; the pass-2 feature loaders reach the same file // through this gate, so it has to ask the same question. - if (!string.Equals(marker, ParquetScoreCache.RECONCILED_SURVIVORS, - StringComparison.Ordinal) || - ParquetScoreCache.IsSubsetWithoutScoreIndex(reconciledPath)) - { + // + // Through the shared predicate rather than re-testing the footer here: + // ScoringTaskShared.CanStreamStage7Join ADMITS a run to the per-run fold on + // exactly this question, and a second copy of it is the drift that admits a + // run to a fold this refusal then aborts. + if (!ParquetScoreCache.IsCurrentReconciledSurvivorSubset(reconciledPath)) stale.Add(fileName); - } } return stale; } diff --git a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs index 69998184a6..239fb7a34c 100644 --- a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs @@ -2807,6 +2807,66 @@ private static void AssertBitEqual(double expected, double actual, string label) BitConverter.DoubleToInt64Bits(actual), label + " bit mismatch"); } + /// + /// The survivor-subset currency predicate + /// (), against real + /// artifacts rather than a hand-built footer. + /// + /// It answers whether a reconciled parquet can be READ in the shape a per-run + /// survivor rebuild needs, and two callers ask it: the Stage 7 refusal that names the + /// stale files, and the admission that decides whether the second-pass join may fold a + /// run at a time. Both of them turn a wrong answer into a whole-cohort outcome - a run + /// admitted to a fold it then aborts, or an O(files) pool nobody asked for - so the + /// three states are pinned here: absent, present-without-the-marker (the Stage 4 + /// original, which is exactly the file a path-derivation slip would hand it), and + /// present-and-current. + /// + [TestMethod] + public void TestIsCurrentReconciledSurvivorSubset() + { + string dir = Path.Combine(Path.GetTempPath(), + "osprey_recon_current_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + string originalPath = Path.Combine(dir, "sample1.scores.parquet"); + string reconciledPath = Path.Combine(dir, "sample1.scores-reconciled.parquet"); + + // Absent: false, and no throw. This is the no-work state on a cold cohort, so it + // has to be an answer rather than an error. + Assert.IsFalse(ParquetScoreCache.IsCurrentReconciledSurvivorSubset(reconciledPath)); + + var original = new List(); + foreach (uint id in new uint[] { 3, 1, 2 }) + original.Add(MakeStreamEntry(id, id * 10.0)); + ParquetScoreCache.WriteScoresParquet(originalPath, original, null, null, "f.mzML"); + + // The Stage 4 original is a well-formed parquet with neither the marker nor the + // score_index column, and it must not pass: it is the file a stem-derivation + // slip substitutes, and its rows are the PRE-reconciliation ones. + Assert.IsFalse(ParquetScoreCache.IsCurrentReconciledSurvivorSubset(originalPath)); + + // Written the way Stage 6 writes it - the marker in the footer, score_index in + // the schema - which is the only combination that passes. + var metadata = new Dictionary + { + { "osprey.reconciled", ParquetScoreCache.RECONCILED_SURVIVORS } + }; + ParquetScoreCache.StreamReconciledScoresParquet( + originalPath, reconciledPath, null, null, metadata, null, "f.mzML", null, + null, null); + Assert.IsTrue(ParquetScoreCache.IsCurrentReconciledSurvivorSubset(reconciledPath)); + + // And it is the POSITIVE form of the interim-shape refusal, not a second + // opinion: a file this accepts is one that one rejects. + Assert.IsFalse(ParquetScoreCache.IsSubsetWithoutScoreIndex(reconciledPath)); + } + finally + { + try { Directory.Delete(dir, true); } catch { /* best-effort */ } + } + } + /// /// Stage-6 streaming reconciled transfer: streaming the original parquet /// group-by-group with an overlay map + gap-fill list diff --git a/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs b/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs index 932da8e882..71d7d12767 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs @@ -21,6 +21,9 @@ * limitations under the License. */ +using System; +using System.Collections.Generic; +using System.IO; using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using pwiz.Osprey.Core; @@ -207,6 +210,12 @@ public void TestResidentPoolGuardError() // buffer SecondPassFDR rebuilds was refused by neither and no token could name it. AssertStage7JoinGuard(); + // The Stage-7 join ADMISSION, which is what decides whether that guard has a + // subject at all. It used to be config.ExpectReconciledInput - one CLI flag - and + // is now the disk question that flag stood in for, so the rules it must not lose + // are pinned here rather than left to the end-to-end gate. + AssertStage7StreamAdmission(); + // The trigger SET itself, not just the message it produces. Each of these takes the // O(files) resident pool and so arms the guard above. AssertNeedsResidentPool(true, fdrbench1); @@ -283,6 +292,40 @@ private static void AssertStage7JoinGuard() streamingAvailable: false, stage7Stream: false, allowUnfixedResident: null)); } + /// + /// The all-runs reconciled-parquet admission: ALL, never any, and never vacuously + /// true. + /// + /// The fold rebuilds each run from its own reconciled parquet, so one run without + /// a readable one is a run it cannot produce - and "some run has one" would fail at that + /// run, hours in. An empty or absent input list is refused for the opposite reason: + /// there is no pool to bound, so vacuous truth would admit a fold over nothing on a + /// configuration no other term in the predicate examines. + /// + /// Only the negative half is asserted here. The positive one needs real Stage 6 + /// artifacts, which is TestIsCurrentReconciledSurvivorSubset's job per file and + /// the regression gate's per cohort; what CANNOT be seen there is a predicate that says + /// yes when it has been handed nothing. + /// + private static void AssertStage7StreamAdmission() + { + Assert.IsFalse(ScoringTaskShared.AllReconciledParquetsCurrent(new OspreyConfig())); + Assert.IsFalse(ScoringTaskShared.AllReconciledParquetsCurrent( + new OspreyConfig { InputFiles = new List() })); + // Paths under a directory that does not exist: every run is missing its parquet, + // which is the cold cohort's state before Stage 6 has written any. + string absent = Path.Combine(Path.GetTempPath(), + "osprey_no_such_dir_" + Guid.NewGuid().ToString("N")); + Assert.IsFalse(ScoringTaskShared.AllReconciledParquetsCurrent( + new OspreyConfig + { + InputFiles = new List + { + Path.Combine(absent, "a.mzML"), Path.Combine(absent, "b.mzML") + } + })); + } + private static void AssertStage6HandoffGuard() { // Streaming: no error, whatever the token says. diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index 3897730ebb..89f50752a7 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -1115,17 +1115,34 @@ the text says so rather than describing the current shape as though it were the survivors instead of rebuilding one run at a time through `StreamFiles`. Both arms are required to produce identical bytes. - **It is NOT the in-place A/B its Stage 6 sibling is, and must not be described as one.** - `CanStreamStage7Join` short-circuits on `!config.ExpectReconciledInput` *before* it reads - the switch, and that flag is set only for `--task SecondPassFDR`. So on a straight-through - run the switch changes nothing - while `SecondPassFdrTask.ValidityKey` appends - `;stage7stream=0` unconditionally, invalidating the `.blib` and every 2nd-pass sidecar and - forcing a full Stage 7 re-run for a setting that cannot change the arm. Comparing the two - shapes means comparing two `--task SecondPassFDR` runs over the same linked bed. + **It IS the in-place A/B its Stage 6 sibling is, and it did not used to be.** + `CanStreamStage7Join` opened on `!config.ExpectReconciledInput`, a flag only + `--task SecondPassFDR` sets, so on a straight-through run the switch changed nothing - + while `SecondPassFdrTask.ValidityKey` appended `;stage7stream=0` regardless, forcing a + full Stage 7 re-run for a setting that could not change the arm. That term is now the + question it stood in for: does every run have a `.scores-reconciled.parquet` on disk in + the survivor-subset shape (`ScoringTaskShared.AllReconciledParquetsCurrent`). Asked of + the disk, it is route-independent - a straight-through run's Stage 6 has just written + those parquets - so the cold run, both resume arms and the `--task SecondPassFDR` merge + all fold run by run, and the switch compares two arms of whichever one you are running. + + The per-run source is not one implementation reached four ways: each arm hands the fold + the per-file half of the whole-run loop it would otherwise have run + (`PerFileRescoreTask.BuildRunPerRunSource` / `BuildResumePerRunSource` / + `BuildStage7PerRunSource`), so run-at-a-time is the same work in the same order as + all-runs-at-once. That is why the arms are required to produce identical bytes, and why + an arm is a call-shape change rather than a second algorithm. + + One route still cannot stream: a pass-2 mode whose per-file half has no worker + (`OSPREY_PASS2_QVALUE=transfer` still competes over the whole pool in Stage 7). Until + `TransferOneFile` moves into `Pass2PerFileWorker`, `Stage7ResidentGuardError` keeps its + `streamingAvailable` exemption - a run with no streamed alternative has no choice for a + token to record. Because nothing in the output distinguishes the arms, the shape that ran is asserted from - the marker line `Second-pass join: folding over N run(s)` rather than inferred - which is - what mode 3 does, scoped to the configurations that can actually stream. + the marker line `Second-pass join: folding over N run(s)` rather than inferred - + `regression.ps1` demands it per leg (the cold run, both resumes, and mode 3's phase 4), + scoped to the configurations that can actually stream. 5. **Whether the 500-run / 64 GB target is met.** It is not yet, and which stage binds is itself moving as each is fixed. The two TODOs above carry the current measurements; diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 6c1dd1b00f..fea4e2271f 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -302,7 +302,16 @@ $knownResidentGaps = @( # the 91.1 GB private measured on the 446-run CHS cohort (2026-09-08), which is the # first endpoint past 82 files. Quoted as a check on the model, not as a second model - # see the note above about three numbers no single model reproduced. - Legs = 'Every leg EXCEPT the streamed Stage-7 join (mode 3''s SecondPassFDR phase, which sets ExpectReconciledInput). ~4.4 GB library + 0.197 GB/file live post-GC: ~20 GB at 82 files, ~103 GB projected at 500, and 92.3 GB predicted vs 91.1 GB measured at 446.' + # The set has SHRUNK from "every leg except mode 3's join phase" to one pass-2 mode. + # CanStreamStage7Join's first term was config.ExpectReconciledInput, which only + # --task SecondPassFDR sets, so the ordinary run could not stream BY CONSTRUCTION; + # derived from the reconciled parquets on disk it is route-independent, and the cold + # run and both resume arms now fold run by run (asserted per leg, not projected). + # What is left is the pass-2 mode with no per-file worker: transfer still computes + # its per-file half in Stage 7, over the whole pool. Moving TransferOneFile into + # Pass2PerFileWorker is what empties this row - and then the guard's + # streamingAvailable exemption has no subject either, so the two go together. + Legs = 'ONLY a pass-2 mode with no per-file worker (OSPREY_PASS2_QVALUE=transfer, i.e. mode 10''s transfer arm). Every default leg - cold straight-through, both resumes, and mode 3''s SecondPassFDR phase - folds run by run. ~4.4 GB library + 0.197 GB/file live post-GC where it is still taken: ~20 GB at 82 files, and 92.3 GB predicted vs 91.1 GB measured at 446.' } ) # Reachable only outside this gate, tokened, each with an open issue: @@ -1156,6 +1165,12 @@ $firstPassFdrRehydrateMarker = 'Resume rehydrate: streaming the first-pass bundl # Asserting the outcome alone is exactly what let an earlier resume fix report success while # testing the old path (defect (b2), TODO-20260901_osprey_firstpassfdr_resume). $perRunHydrateMarker = 'Per-run rescore: hydrating each of' +# The line every arm of the streamed Stage-7 join logs - the reconciled-input merge, the +# straight-through cold run and the straight-through resume all open with these words on +# purpose. Output is IDENTICAL whichever arm runs, so this line is the only evidence the +# bounded join happened at all; giving each arm its own wording would need three markers +# and would let a fourth arm ship unwatched. +$stage7StreamMarker = 'Second-pass join: folding over ' # FirstPassFDR's half of the same shape: on a rehydrate where the analysis-wide summary is on # disk it publishes the survivor loader and builds no experiment-wide bundle, so it emits this @@ -1737,6 +1752,23 @@ foreach ($name in $selected) { $dataFp = Get-DirFingerprint -Dir $inputs.Dir $straightDir = Join-Path $runRoot "$name\straight" + # Whether this run was ASKED for a configuration that cannot stream the Stage-7 join, so + # the legs below can tell "took the resident join" from "was told to". These are + # CanStreamStage7Join's OWN terms, not a mode list: the switch that forces the resident + # join, the two that make NeedsResidentPool true, and any pass-2 mode other than + # protein-compact (transfer still computes its per-file half in Stage 7). Enumerated + # rather than inferred from the log, because a resident run says nothing about WHY it was + # resident - and a silent SKIP for the wrong reason is what these legs exist to prevent. + # + # ExpectReconciledInput is not among them, and used to be the whole answer: it was + # CanStreamStage7Join's first term, and only --task SecondPassFDR sets it, so the ordinary + # run could not stream by construction. Deriving the admission from the reconciled parquets + # on disk is what puts every leg under one question. + $cannotStreamJoin = + ($env:OSPREY_STAGE7_STREAM -eq '0') -or + ($env:OSPREY_FDR_PROJECTION -eq '0') -or + (-not [string]::IsNullOrWhiteSpace($env:OSPREY_PASS2_QVALUE) -and + $env:OSPREY_PASS2_QVALUE -ne 'protein-compact') $proteinDump = Join-Path $straightDir 'cs_stage7_protein_fdr.tsv' # GoldenFolder, not Folder: StellarLibDecoy shares the stellar mzML folder, # so keying the golden on Folder alone would collide with Stellar's. @@ -2047,21 +2079,12 @@ foreach ($name in $selected) { # needs a narrower rule should name the condition rather than restore a blanket skip. # ...unless the run was asked for a configuration that cannot stream, in which case it # is doing exactly what it was told and demanding the marker would fail it for - # complying. These are CanStreamStage7Join's OWN terms, not a mode list: the switch that - # forces the resident join, the two that make NeedsResidentPool true, and any pass-2 - # mode other than protein-compact (transfer still computes its per-file half in Stage 7). - # ExpectReconciledInput is not among them because phase 4 always sets it. - # Enumerated rather than inferred from the log, because a resident run says nothing - # about WHY it was resident - and a silent SKIP for the wrong reason is what this leg - # exists to prevent. - $chainCannotStream = - ($env:OSPREY_STAGE7_STREAM -eq '0') -or - ($env:OSPREY_FDR_PROJECTION -eq '0') -or - (-not [string]::IsNullOrWhiteSpace($env:OSPREY_PASS2_QVALUE) -and - $env:OSPREY_PASS2_QVALUE -ne 'protein-compact') + # complying. $cannotStreamJoin, computed once per dataset above, is that question; it + # governs every leg's copy of this assertion rather than each one enumerating the + # terms again. $chainStreamed = Select-String -Path (Join-Path (Join-Path $chainRoot 'logs') 'phase4.log') ` -Pattern 'Second-pass join: folding over \d+ run\(s\)' -Quiet - if ($chainCannotStream) { + if ($cannotStreamJoin) { $summaryLines.Add("$name mode3 (streamed join): SKIP (this configuration cannot stream the join)") } elseif (-not $chainStreamed) { $overallFail = $true @@ -2646,6 +2669,54 @@ foreach ($name in $selected) { } } + # ---- mode 1/2/5: the IN-PROCESS legs took the streamed Stage-7 join ---- + # Here, beside mode 6, for mode 6's own reason: it reads the logs of the legs above and + # they have to have been written. Mode 3's chain carries the same assertion inside its own + # block, where its phase-4 log lives. + # + # This could not be asserted before. CanStreamStage7Join's admission WAS + # config.ExpectReconciledInput, which only --task SecondPassFDR sets, so every in-process + # leg was resident by construction and demanding the marker would have failed each of them + # for behaving correctly. Derived from the reconciled parquets on disk the admission is + # route-independent, and these three legs - the cold run and the two resumes - are exactly + # the shapes an operator runs. The 91.1 GB measured on a 446-run cohort was an ORDINARY + # `-i ... --output-dir` resume, i.e. the third of them. + # + # Three separate arms, not one: the cold run publishes its source from + # PerFileRescoreTask.Run, and the two resumes from its Rehydrate, having arrived there by + # different routes (mode 2 re-runs FirstPassFDR, mode 5 rebuilds its bundle from its own + # sidecars, which leaves the survivor lists POPULATED where mode 2 leaves them released). + # A marker on one says nothing about the others. + # + # A leg that did not run is SKIP, not FAIL - -SkipResume / -SkipRehydrate are legitimate. + foreach ($streamLeg in @( + @{ Log = 'straight.log'; Mode = 'mode1'; What = 'the cold straight-through run' }, + @{ Log = 'resume.log'; Mode = 'mode2'; What = 'the resume' }, + @{ Log = 'rehydrate.log'; Mode = 'mode5'; What = 'the own-sidecar rehydrate' })) { + $legPath = Join-Path $straightDir $streamLeg.Log + if ($cannotStreamJoin) { + $summaryLines.Add(("$name $($streamLeg.Mode) (streamed join): SKIP " + + '(this configuration cannot stream the join)')) + } elseif (-not (Test-Path -LiteralPath $legPath)) { + $summaryLines.Add("$name $($streamLeg.Mode) (streamed join): SKIP (leg not run)") + } else { + $legStream = Test-LogMarker -LogPath $legPath -Marker $stage7StreamMarker ` + -Description ("$($streamLeg.What) folding Stage 7 one run at a time instead " + + 'of rebuilding every run''s survivors at once') + if ($legStream.Pass) { + $summaryLines.Add(("$name $($streamLeg.Mode) (streamed join): PASS " + + '(per-run fold, no all-runs pool)')) + } else { + $overallFail = $true + Write-Problem-Tc ("$name $($streamLeg.Mode) (streamed join): FAIL - Stage 7 " + + 'built the whole-run survivor pool. Output is unchanged either way; only ' + + 'this line distinguishes them.') + $legStream.Issues | ForEach-Object { Write-Host " $_" -ForegroundColor Red } + $summaryLines.Add("$name $($streamLeg.Mode) (streamed join): FAIL") + } + } + } + # ---- mode 6: the library-fragment release engaged on every leg that holds the library ---- # Runs LAST because it reads the logs of all the legs above -- straight-through, # resume, and every phase of the HPC chain -- and they have to have been written. From f7920db59e0a0d8f0a8cef1dfa8fa1eefaf692c6 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 15:24:35 -0700 Subject: [PATCH 25/30] Retired --input-scores, the Rust-era input-kind seam * Deleted the flag, OspreyConfig.InputScores and Program.ResolveInputScores * Made every task take -i and derive each run's parquets from the input stem * Collapsed the membership predicates onto --task, which was the only seam deciding them * Accepted an absent input whose scores parquet is on disk - a staged join node's state See TODO-20260908_osprey_input_scores_retirement.md in pwiz-ai/todos Co-Authored-By: Claude --- .../Documentation/Help/en/CommandLine.html | 15 +- pwiz_tools/Osprey/Osprey.Core/OspreyConfig.cs | 27 +- .../Osprey/Osprey.Tasks/BlibOutputWriter.cs | 7 +- .../Osprey/Osprey.Tasks/FirstPassFdrTask.cs | 20 +- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 44 +-- .../Osprey/Osprey.Tasks/PerFileScoringTask.cs | 99 +++--- .../Osprey/Osprey.Tasks/PipelineContext.cs | 2 +- .../Osprey/Osprey.Tasks/RescoreHydration.cs | 14 + .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 72 +++- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 12 +- .../Osprey.Test/LibraryFragmentReleaseTest.cs | 36 +- .../Osprey.Test/PipelineMembershipTest.cs | 45 ++- pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs | 311 ++---------------- .../Osprey.Test/ResidentPoolGuardTest.cs | 19 +- pwiz_tools/Osprey/Osprey/AnalysisPipeline.cs | 27 +- pwiz_tools/Osprey/Osprey/OspreyCommandArgs.cs | 53 ++- pwiz_tools/Osprey/Osprey/Program.cs | 236 +++++-------- pwiz_tools/Osprey/Osprey/RescoreWorker.cs | 6 +- .../Osprey/docs/00-pipeline-architecture.md | 8 +- .../Osprey/docs/11-boundary-overrides.md | 2 +- .../Osprey/docs/15-hpc-scoring-split.md | 41 ++- pwiz_tools/Osprey/docs/16-determinism.md | 6 +- pwiz_tools/Osprey/docs/19-testing.md | 4 +- pwiz_tools/Osprey/docs/20-command-line.md | 24 +- pwiz_tools/Osprey/docs/DIVERGENCES.md | 2 +- pwiz_tools/Osprey/docs/README.md | 4 +- pwiz_tools/Osprey/regression.ps1 | 18 +- 27 files changed, 453 insertions(+), 701 deletions(-) diff --git a/pwiz_tools/Osprey/Documentation/Help/en/CommandLine.html b/pwiz_tools/Osprey/Documentation/Help/en/CommandLine.html index d53c9dcf58..8508825c4c 100644 --- a/pwiz_tools/Osprey/Documentation/Help/en/CommandLine.html +++ b/pwiz_tools/Osprey/Documentation/Help/en/CommandLine.html @@ -68,7 +68,6 @@

Osprey command-line usage

-
ArgumentDescription
--task <SpectraCache | PerFileScoring | FirstPassFDR | PerFileRescoring | SecondPassFDR | ModelDiagnostics>HPC: run exactly one pipeline task (one node = one task). Omit for the full pipeline. SpectraCache stages the .spectra.bin caches; ModelDiagnostics regenerates only the --model-diagnostics report for a COMPLETED run, writing no other artifact.
--input-scores <paths|dir>HPC: one or more .scores.parquet files, or a single directory (non-recursive). Mutex with --input.
Logging
@@ -87,22 +86,22 @@

Osprey command-line usage

-h, --helpShow this help message ([ascii|unicode|sections|html|<Section>])
-v, --versionShow version
-

EXAMPLES:

osprey -i sample.mzML -l library.tsv -o results.blib

osprey -i *.mzML -l library.tsv -o results.blib --resolution hram

HPC SPLIT (one node = one --task): see --task / --input-scores above.

Distributed execution (HPC)
+

EXAMPLES:

osprey -i sample.mzML -l library.tsv -o results.blib

osprey -i *.mzML -l library.tsv -o results.blib --resolution hram

HPC SPLIT (one node = one --task): see --task above.

Distributed execution (HPC)

Run with no --task for the whole pipeline in one process. For distributed (HPC / workflow-engine) execution the pipeline splits at its join / fan-out boundaries into four single-task workers — one node = one --task: PerFileScoring (split, per file) → FirstPassFDR (join, all files) → PerFileRescoring (split, per file) → SecondPassFDR (join, all files). Pass the same --library and search options to every task; the parquet integrity check rejects inputs whose search/library hash does not match.

 # split 1 - one process per mzML (writes <stem>.scores.parquet, <stem>.calibration.json beside each input)
 Osprey --task PerFileScoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01
 
-# join 1 - one process over ALL parquets (pass a directory so the order is deterministic)
-Osprey --task FirstPassFDR --input-scores ./scores_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01
+# join 1 - one process over ALL runs (pass a sorted list so the order is deterministic)
+Osprey --task FirstPassFDR --input-list runs.txt -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01
 #   writes beside each parquet: <stem>.1st-pass.fdr_scores.bin, <stem>.reconciliation.json
 
 # split 2 - one process per file (parquet + its two sidecars co-located)
-Osprey --task PerFileRescoring --input-scores s1.scores.parquet -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01
+Osprey --task PerFileRescoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01
 #   writes: <stem>.scores-reconciled.parquet
 
-# join 2 - one process over ALL reconciled parquets (writes out.blib)
-Osprey --task SecondPassFDR --input-scores ./reconciled_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01
+# join 2 - one process over ALL runs, reading their reconciled parquets (writes out.blib)
+Osprey --task SecondPassFDR --input-list runs.txt -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01
-

--input-scores takes a directory (globbed and sorted internally) or an explicit file list (used in the order given). FirstPassFDR reconciliation is order-sensitive, so for FirstPassFDR and SecondPassFDR pass a directory or a deterministically sorted list. The rehydration sidecars must travel with their parquet into each worker's working directory. Let the scheduler do the fan-out (one file per split process) rather than --parallel-files, which is the single-node multi-file mode.

+

EVERY task takes -i, naming the DATA files - the same names the first split was given. A join task derives each run's parquet and sidecars from the input stem, so the data file itself need not still exist: what has to be in the worker's working directory (or under --output-dir) is that run's artifacts. FirstPassFDR reconciliation is order-sensitive, so pass a deterministically sorted list - --input-list takes one path per line and is what a cohort past a few hundred runs needs, since -i spends the command line at O(files). Let the scheduler do the fan-out (one file per split process) rather than --parallel-files, which is the single-node multi-file mode.

\ No newline at end of file diff --git a/pwiz_tools/Osprey/Osprey.Core/OspreyConfig.cs b/pwiz_tools/Osprey/Osprey.Core/OspreyConfig.cs index b5a995b5f4..2704e12f3f 100644 --- a/pwiz_tools/Osprey/Osprey.Core/OspreyConfig.cs +++ b/pwiz_tools/Osprey/Osprey.Core/OspreyConfig.cs @@ -328,22 +328,15 @@ public class OspreyConfig /// Pipeline-membership flag (read by each task's IsIncluded): /// include only the per-file fan-out, not the joining tasks. Set by both /// --task PerFileScoring and --task PerFileRescoring; the - /// concrete behavior depends on the input type. With -i mzML it - /// is the Stage 1-4 worker — each input produces a - /// {stem}.scores.parquet next to it, no FDR, no blib. With - /// it is the Stage 6 rescore worker. The two - /// are told apart by input type (see ). + /// concrete behavior depends on which of the two selected it. + /// PerFileScoring is the Stage 1-4 worker - each input produces a + /// {stem}.scores.parquet next to it, no FDR, no blib; + /// PerFileRescoring is the Stage 6 rescore worker. The two are told apart + /// by , which is the only thing that ever decided it - + /// they used to be told apart by input KIND as well, and that second seam is gone. /// public bool NoJoin { get; set; } - /// - /// HPC scoring split: when set (non-null, non-empty), skip Stages 1-4 - /// entirely and load these per-file scoring caches as the starting - /// point for Stage 5+. Set by --input-scores. When set, - /// is ignored. - /// - public List InputScores { get; set; } - /// /// HPC: when true, exit after Stage 5 + reconciliation planning, /// having written the boundary files @@ -371,10 +364,10 @@ public class OspreyConfig /// membership flags above (, /// , ) /// are derived from this and drive each task's IsIncluded; this - /// property additionally lets argument validation enforce the - /// task↔input-type contract (e.g. PerFileScoring takes mzML, - /// PerFileRescore takes ) and name the task the - /// user actually typed in error messages. + /// property additionally lets argument validation name the task the user actually + /// typed in error messages. It no longer has an input-KIND contract to enforce: + /// every task takes the same data files, and the second seam that said "you handed + /// me parquets, so Stage 1-4 is done" has retired into these flags. /// public HpcTask? SelectedTask { get; set; } diff --git a/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs b/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs index c5f3636907..e4c4edc336 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs @@ -130,9 +130,10 @@ private static Dictionary CreateSourceFiles( IReadOnlyList fileNames, double fdrThreshold) { string libraryIdName = Path.GetFileName(config.LibrarySource.Path); - var inputs = config.InputScores != null && config.InputScores.Count > 0 - ? config.InputScores.ConvertAll(RescoreHydration.SyntheticInputFromParquet) - : config.InputFiles; + // The data files, on every route. A Stage 7 node used to be handed parquets and + // had to convert them back into data-file names right here, to write the + // SpectrumSourceFiles rows Skyline reads; it is handed the names themselves now. + var inputs = config.InputFiles; var sourcePathByName = new Dictionary(); if (inputs != null) { diff --git a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs index 26a17e0ae5..b562d6b000 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/FirstPassFdrTask.cs @@ -94,18 +94,14 @@ internal sealed class FirstPassFdrTask : OspreyTask /// internal static bool IsIncludedFor(OspreyConfig c) { - bool inputs = c.InputScores != null && c.InputScores.Count > 0; - // The (inputs && StopAfterStage5) clause leans on a CLI-enforced - // invariant: StopAfterStage5 is set by --task FirstPassFDR, which - // requires --input-scores, so StopAfterStage5 implies inputs at - // parse time -- a --task FirstPassFDR run can never reach here without - // InputScores. - // ProgramTests.TestValidateFirstPassFdrRequiresInputScores pins that - // rejection, since the membership truth table (PipelineMembershipTest) - // does not encode the cross-flag dependency on its own. - return (!inputs && !c.NoJoin) - || (inputs && c.StopAfterStage5) - || (inputs && !c.NoJoin && !c.ExpectReconciledInput); + // Three clauses over two seams collapsed to one over the task flags. The + // retired term was `inputs` - were parquets supplied - which the truth table + // above shows was never doing independent work: it tracked exactly the tasks + // whose flags already say so. Excluded for the two per-file workers (NoJoin) + // and for the Stage 7 node (ExpectReconciledInput); included for the full + // pipeline, for --task FirstPassFDR itself, and for --task ModelDiagnostics, + // which needs first-pass state to render. + return !c.NoJoin && !c.ExpectReconciledInput; } // Stage 5/6 planning byproducts this task publishes. The same four types diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index 8fdc2d62db..b32b3ff95b 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -151,26 +151,28 @@ private readonly Dictionary> _resetEntryIdsByFile public override string Name => TASK_NAME; /// - /// Computes the Stage 6 rescore in straight-through, the rescore worker - /// (--task PerFileRescoring), and the --input-scores - /// full-pipeline. Excluded in --task PerFileScoring, --task FirstPassFDR (stops at Stage 5), - /// and the --task SecondPassFDR run (where it rehydrates rather than - /// re-scoring, SecondPassFDR having no mzMLs). + /// Computes the Stage 6 rescore in the straight-through run and in the rescore + /// worker (--task PerFileRescoring). Excluded in --task PerFileScoring, + /// --task FirstPassFDR (stops at Stage 5), --task ModelDiagnostics (a render, which + /// also stops there) and --task SecondPassFDR, where it rehydrates rather than + /// re-scoring - that node has no data files to score from. /// public override bool IsIncluded(PipelineContext ctx) { var c = ctx.Config; - bool inputs = c.InputScores != null && c.InputScores.Count > 0; - // StopAfterStage5 is checked on BOTH input routes. It used to appear only in the - // --input-scores clause, which was enough while --task FirstPassFDR was the only - // thing that set it - that task rejects -i. --task ModelDiagnostics also stops - // after Stage 5 and takes -i, so this task ran anyway, demanded CompactedEntries - // that a diagnostics-only fold never publishes, and failed the run AFTER the report - // it was asked for had been written. A flag named for a stage boundary has to mean - // that boundary whatever the inputs look like. - return (!inputs && !c.NoJoin && !c.StopAfterStage5) - || (inputs && c.NoJoin) - || (inputs && !c.NoJoin && !c.StopAfterStage5 && !c.ExpectReconciledInput); + // The rescore worker is the ONE task NoJoin does not distinguish - it is set by + // --task PerFileScoring too - and the input KIND is what used to tell them + // apart: mzML in meant Stage 1-4, parquets in meant Stage 6. Both are named by + // their data files now, so the task says which worker this is, which is the only + // thing that ever actually decided it. + // + // StopAfterStage5 means that boundary whatever the inputs look like: it used to + // be checked on one route only, which was enough while --task FirstPassFDR was + // its only setter, and --task ModelDiagnostics sets it too - so this task ran + // anyway, demanded CompactedEntries that a diagnostics fold never publishes, and + // failed the run AFTER the report it was asked for had been written. + return c.SelectedTask == HpcTask.PerFileRescore + || (!c.NoJoin && !c.StopAfterStage5 && !c.ExpectReconciledInput); } // The final milestone of the shared mutable entry buffer: this task @@ -2021,12 +2023,10 @@ private static Func BuildPerRunHydrate( // and "the gate is green so the new path must have run" is precisely the inference // that let an earlier resume fix report success while testing the old path (defect // (b2), TODO-20260901_osprey_firstpassfdr_resume). A test can assert this line. - // Count the runs from the published parquet paths, NOT from config.InputScores - - // that list is null on a straight-through run, where the inputs are mzMLs. It read - // InputScores while the predicate above still required it; dropping that term left - // this line behind, and it threw a NullReferenceException on the first - // straight-through run. The paths are the right source on both shapes anyway: they - // are what the loop iterates. + // Count the runs from the published parquet paths. It used to read InputScores, + // which was null on a straight-through run, and it threw a NullReferenceException + // on the first one after the predicate above stopped requiring that list. The + // paths are the right source regardless: they are what the loop iterates. ctx.LogInfo(string.Format( @"Per-run rescore: hydrating each of {0} run(s) from its own artifacts " + @"(no all-runs pre-load; {1} retained base_id(s) read once).", diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs index 6dd4808bf4..a14bba5d7d 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs @@ -77,15 +77,14 @@ internal sealed class PerFileScoringTask : OspreyTask public override string Name => @"PerFileScoring"; /// - /// Computes per-file scores from spectra only when no per-file scores - /// were supplied via --input-scores. Under --input-scores it is - /// excluded: a downstream task lazy-rehydrates the supplied scores - /// through ctx.Demand<PerFileScoringTask>(). + /// Computes per-file scores from spectra for every task except the three that + /// start after Stage 4. For those it is excluded and a downstream task + /// lazy-rehydrates each run's scores through + /// ctx.Demand<PerFileScoringTask>(). /// public override bool IsIncluded(PipelineContext ctx) { - bool inputs = ctx.Config.InputScores != null && ctx.Config.InputScores.Count > 0; - return !inputs; + return !ScoringTaskShared.StartsAfterPerFileScoring(ctx.Config); } // Stage 1-4 byproducts this task publishes for downstream consumers to @@ -394,7 +393,7 @@ public override bool Run(PipelineContext ctx) // // NOT reachable today, and the claim that it was is wrong: Program.cs rejects // --task SecondPassFDR combined with --input and requires --input-scores, so - // ExpectReconciledInput implies InputScores.Count > 0 and IsIncluded returns false - + // ExpectReconciledInput means this task is excluded and IsIncluded returns false - // Run is never entered on that config. This is aligned with its two siblings so the // one decision has one predicate, not so that a live defect is closed. bool needsResidentPool = !CanUseLeanProjection(ctx.Config, hasReconSidecars: false, @@ -496,12 +495,13 @@ public override bool Rehydrate(PipelineContext ctx) // (CanRehydrate), and a downstream task is the first to touch its // state. Load those valid parquets straight from disk (never // compute) so Rehydrate stays pure -- Run is outer-loop-only. The - // worker-mode join-only disk-load below applies only when - // --input-scores actually supplied the per-file scores. - if (ctx.Config.InputScores == null || ctx.Config.InputScores.Count == 0) + // worker-mode join-only disk-load below applies only to the tasks that + // start after Stage 4. That used to be asked as "were parquets supplied"; + // one seam now answers it, and it is the task. + if (!ScoringTaskShared.StartsAfterPerFileScoring(ctx.Config)) return RehydrateFromOwnOutputs(ctx); - // Disk-load path for worker-mode entry (--input-scores): the + // Disk-load path for a node that starts after Stage 4: the // per-file Stage 2-4 scores already exist on disk, so load the // FdrEntry stubs + PIN features straight from the parquets // (Stage 1 library still loads -- Stage 5+ needs it) instead of @@ -524,12 +524,7 @@ public override bool Rehydrate(PipelineContext ctx) // knows each input parquet path and fills this in. var perFileParquetPaths = new Dictionary(); - int nFiles = config.InputScores.Count; - - // InputFiles is synthesized from the --input-scores parquet stems - // once at pipeline entry (AnalysisPipeline.Run), so downstream code - // (Stage 6 rescore's fileNameToIdx in particular) already has the - // synthetic input paths by the time this load runs. + int nFiles = config.InputFiles.Count; // Mirror Run's EffectiveFileParallelism bookkeeping via the shared // resolver (unused by the disk-load path, which never calls @@ -688,7 +683,7 @@ private bool RehydrateFromOwnOutputs(PipelineContext ctx) // This site gated on bare NeedsResidentPool, which no longer excludes // ExpectReconciledInput, so an ExpectReconciledInput config arriving here would // take the lean branch and add an EMPTY entry list per file. Not reachable today - // (Run routes InputScores runs away from Rehydrate), but re-deriving the rule at + // (Run routes a post-Stage-4 task away from Rehydrate), but re-deriving the rule at // one call site and importing it at the other is precisely the drift this change // exists to remove. hasReconSidecars is false here: this path has no bundle. // ARM THE GUARD ON THE SAME DECISION, for the reason the branch below states: the @@ -1303,8 +1298,13 @@ private FdrProjectionSet LoadJoinOnlyScores( PipelineContext ctx) { hydrationFailed = false; + // Each run's parquet, derived from its input stem: the reconciled sibling where + // Stage 6 wrote one, else the Stage 4 file. This list used to arrive ready-made + // on --input-scores, and the pipeline's first act was to convert it BACK into + // input stems so the sidecar helpers could work. + var scoresPaths = ScoringTaskShared.ScoresPathsForInputs(config); // --task FirstPassFDR: load per-file FdrEntry stubs directly from - // each .scores.parquet listed via --input-scores. Skips the + // each run's .scores.parquet, derived from its input stem. Skips the // per-file Stage 2-4 scoring (Stage 1 library load already ran // in Run). Also loads a best-effort calibration JSON sibling // per file (the loop below) for Stage 6 reconciliation, like @@ -1313,13 +1313,12 @@ private FdrProjectionSet LoadJoinOnlyScores( // Aborts with a clear, file-named error if the operator points // SecondPassFDR at parquets from a different scoring run. string validationError = ParquetScoreCache.ValidateScoresParquetGroup( - config.InputScores, config, OspreyVersion.Current); + scoresPaths, config, OspreyVersion.Current); if (validationError != null) throw new InvalidDataException(validationError); ctx.LogInfo(string.Format( - @"--input-scores: loading {0} per-file score parquet(s)", - config.InputScores.Count)); + @"Loading {0} per-file score parquet(s)", scoresPaths.Count)); // Lean on the HPC merge/join too (#4400): a large FirstPassFDR node // loading every worker's .scores.parquet used to rebuild the full fat // FdrEntry stubs + PIN features (~53 GB at 82 files) -- the same Stage-5 @@ -1435,7 +1434,7 @@ private FdrProjectionSet LoadJoinOnlyScores( // non-zero exit code, not an unhandled stack trace. _rescoreInputs = HydrateRescoreBundleOrNull( () => RescoreHydration.HydrateCompactedStreaming( - perFileEntries, config.InputScores, + perFileEntries, scoresPaths, (fileIdx, fileName, parquetPath) => LoadJoinOnlyScoresForFile( config, fileIdx, fileName, parquetPath, perFileParquetPaths, perFileCalibrations, perFileIsolationMz, _sequencePool.Value, ctx), @@ -1462,17 +1461,18 @@ private FdrProjectionSet LoadJoinOnlyScores( return null; } - for (int fileIdx = 0; fileIdx < config.InputScores.Count; fileIdx++) + for (int fileIdx = 0; fileIdx < scoresPaths.Count; fileIdx++) { - string parquetPath = config.InputScores[fileIdx]; - // Derive the bare input stem via the single shared suffix-strip - // helper so a .scores-reconciled.parquet input maps to the same - // fileName key as its .scores.parquet sibling (a naive trailing - // ".scores" strip would leave the bogus key ".reconciled"). + string parquetPath = scoresPaths[fileIdx]; + // The input's own stem, not one recovered from the parquet name. The + // recovery existed because the parquet was all this path was given; it had + // to strip ".scores" or ".scores-reconciled" to get back to a key that + // matches the rest of the pipeline, and a naive trailing strip left the + // bogus key ".reconciled". string fileName = Path.GetFileNameWithoutExtension( - RescoreHydration.SyntheticInputFromParquet(parquetPath)) ?? string.Empty; + config.InputFiles[fileIdx]) ?? string.Empty; ctx.LogInfo(string.Format(@"Loading file {0}/{1}: {2} (from {3})", - fileIdx + 1, config.InputScores.Count, fileName, parquetPath)); + fileIdx + 1, scoresPaths.Count, fileName, parquetPath)); if (builder != null) { // Lean: stream 32 B projection rows straight from the parquet; no @@ -1701,7 +1701,7 @@ private static List LoadJoinOnlyScoresForFile( // of its own child. The counter here is this file within the bundle; the percentage // above it is the bundle's own. ctx.LogInfo(string.Format(@" Loading file {0}/{1}: {2} (from {3})", - fileIdx + 1, config.InputScores.Count, fileName, parquetPath)); + fileIdx + 1, config.InputFiles.Count, fileName, parquetPath)); var stubs = ParquetScoreCache.LoadFdrStubsFromParquet(parquetPath, null, sequencePool); // Keep the fail-fast the feature load used to provide: a foreign or truncated // parquet missing the PIN schema must stop here, not surface downstream. @@ -1752,13 +1752,15 @@ private static void LoadJoinOnlyPerRunNames( string consumer, PipelineContext ctx) { + var scoresPaths = ScoringTaskShared.ScoresPathsForInputs(config); ctx.LogInfo(string.Format( - @"--input-scores: {0} run(s) will be hydrated one at a time by {1}; " + - @"no all-runs pre-load.", config.InputScores.Count, consumer)); - foreach (string parquetPath in config.InputScores) + @"{0} run(s) will be hydrated one at a time by {1}; " + + @"no all-runs pre-load.", scoresPaths.Count, consumer)); + for (int i = 0; i < scoresPaths.Count; i++) { + string parquetPath = scoresPaths[i]; string fileName = Path.GetFileNameWithoutExtension( - RescoreHydration.SyntheticInputFromParquet(parquetPath)) ?? string.Empty; + config.InputFiles[i]) ?? string.Empty; perFileEntries.Add(new KeyValuePair>(fileName, new List())); perFileParquetPaths[fileName] = parquetPath; LoadJoinOnlyCalibration(fileName, parquetPath, perFileCalibrations, @@ -1767,20 +1769,16 @@ private static void LoadJoinOnlyPerRunNames( } /// - /// The --input-scores per-file names in input order, each derived from its - /// parquet stem through the same shared suffix-strip helper - /// 's resident loop and - /// use, so index i here names - /// the file the streaming hydrate reports at index i. + /// The per-file names in input order, so index i here names the file the streaming + /// hydrate reports at index i. Straight off the input stems: the shared suffix-strip + /// helper this used to route through existed only to recover a stem from a parquet + /// name, which is the round trip --input-scores forced. /// private static string[] JoinOnlyFileNames(OspreyConfig config) { - var fileNames = new string[config.InputScores.Count]; + var fileNames = new string[config.InputFiles.Count]; for (int i = 0; i < fileNames.Length; i++) - { - fileNames[i] = Path.GetFileNameWithoutExtension( - RescoreHydration.SyntheticInputFromParquet(config.InputScores[i])) ?? string.Empty; - } + fileNames[i] = Path.GetFileNameWithoutExtension(config.InputFiles[i]) ?? string.Empty; return fileNames; } @@ -1900,6 +1898,10 @@ private bool HydrateRescoreBundleIfPresent( } if (hasReconSidecars) { + // Each run's parquet, derived from its input stem - the same derivation the + // loader above makes, from the same source, so the two cannot name different + // files for one run. + var scoresPaths = ScoringTaskShared.ScoresPathsForInputs(config); // Already hydrated when the loader took the file-count-bounded streaming // path (ShouldStreamCompaction): it has to own the hydrate, because the // sidecar overlay and the compaction have to happen inside its per-file @@ -1911,7 +1913,7 @@ private bool HydrateRescoreBundleIfPresent( { _rescoreInputs = HydrateRescoreBundleOrNull( () => RescoreHydration.HydrateReconciliationOverlay( - perFileEntries, config.InputScores, + perFileEntries, scoresPaths, FdrExperimentSidecar.ReadMap( FdrExperimentSidecar.PathFor(config.OutputBlib, ScoringTaskShared.ArtifactSiblingPath(config), FdrScoresSidecar.Pass.FirstPass), @@ -1997,9 +1999,8 @@ private static RescoreInputs HydrateRescoreBundleOrNull( /// private static bool AllHaveReconSidecars(OspreyConfig config) { - foreach (var parquetPath in config.InputScores) + foreach (var syntheticInput in config.InputFiles) { - string syntheticInput = RescoreHydration.SyntheticInputFromParquet(parquetPath); // Version-fenced like every other sidecar gate: a v3 file left by an older build // is present but unreadable, and answering "yes, all sidecars are here" off // File.Exists keeps the fat pool on a path whose overlay then cannot load it. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs b/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs index fd9f634f35..54843e38a9 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs @@ -126,7 +126,7 @@ public sealed class PipelineContext /// reproduce the same hash a straight-through invocation would /// stamp into its parquet footers. Pipeline-populated fields /// that do NOT feed those hashes (e.g. the worker-mode - /// synthesis of InputFiles from InputScores) may be + /// population of InputFiles from --input-list) may be /// written once at pipeline entry. Run-time state that is not parsed /// config (e.g. file parallelism) lives on /// instead. For per-file scratch that diff --git a/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs b/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs index adaa5e8687..68855a062e 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/RescoreHydration.cs @@ -1208,6 +1208,20 @@ private static bool StemsEqual(IList a, IList b) /// reconciliation JSON) without duplicating them. The synthetic /// path is never opened — only its components are inspected. /// Mirrors Rust's synthetic_input_from_parquet. + /// + /// Its REASON is gone. It existed because --input-scores named parquets + /// on the command line, so the pipeline's first act was to convert them back into + /// data-file names for the sidecar helpers - a round trip, and the clearest evidence + /// that the flag was a second way of saying what --task already said. That + /// flag has retired; every task is given the data-file names directly. + /// + /// What is left is internal: the hydrate methods below still take a PARQUET + /// path per run (from PerFileParquetPaths, which is how the pipeline carries + /// them), and derive the stem back from it. Inverting those signatures to take the + /// input and derive the parquet is the remaining half of the retirement - a + /// no-behaviour-change refactor, deliberately not folded into the CLI change so the + /// gate can attribute a failure to one of them. See + /// TODO-20260908_osprey_input_scores_retirement.md. /// public static string SyntheticInputFromParquet(string parquetPath) { diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index 139e4ebcf8..1d8a558820 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -331,19 +331,18 @@ internal static void TallyPreCompaction( /// to, for naming the analysis-wide experiment-scope FDR sidecar /// (). /// - /// Prefers InputScores over InputFiles because a distributed - /// --task node is given its inputs as scores parquets and may have no mzML list - /// at all; both resolve to the same directory, since the parquets are written beside the - /// inputs. What matters is only that every phase of one analysis picks a path that - /// resolves the SAME way - the blib's own directory does not, which is the bug this - /// exists to avoid. + /// The FIRST input, on every route. It used to prefer InputScores, + /// because a distributed --task node was given its inputs as scores parquets + /// and might have had no data-file list at all; every node is given the same list + /// now, and both forms resolved to the same directory anyway since the parquets are + /// written beside the inputs. What matters is only that every phase of one analysis + /// picks a path that resolves the SAME way - the blib's own directory does not, which + /// is the bug this exists to avoid. /// internal static string ArtifactSiblingPath(OspreyConfig config) { if (config == null) return null; - if (config.InputScores != null && config.InputScores.Count > 0) - return config.InputScores[0]; if (config.InputFiles != null && config.InputFiles.Count > 0) return config.InputFiles[0]; return null; @@ -435,6 +434,63 @@ internal static bool CanHydratePerRun(OspreyConfig config) return !string.IsNullOrEmpty(path) && RetainedBaseIdSidecar.IsCurrentFormat(path); } + /// + /// Every task that starts AFTER Stage 4 - the two joins and the rescore worker. They + /// are handed a directory of per-run artifacts rather than spectra, so + /// does not run for them; a consumer materializes + /// its state through ctx.Demand, which routes to its disk load. + /// + /// This used to be asked as "were parquets supplied on the command line", which + /// is the INPUT KIND - the Rust pipeline's way of saying Stage 1-4 was done. The port + /// says it with --task, and the two seams disagreeing is what let + /// --task ModelDiagnostics join the pipeline and demand state a diagnostics + /// fold never publishes. One question, asked of the task. + /// + /// ModelDiagnostics is deliberately NOT here. It is neither a fan-out nor + /// a join but a render over retained products, and it needs the per-file load to have + /// happened - which it did by taking -i even while the others took parquets. + /// That asymmetry was the first symptom of the two seams, and it survives the + /// retirement as an ordinary membership fact rather than as an input-kind accident. + /// + internal static bool StartsAfterPerFileScoring(OspreyConfig config) + { + switch (config.SelectedTask) + { + case HpcTask.FirstPassFdr: + case HpcTask.PerFileRescore: + case HpcTask.SecondPassFdr: + return true; + default: + return false; + } + } + + /// + /// Each input's scores parquet, in input order: the reconciled sibling where Stage 6 + /// has written one, else the Stage 4 file. + /// + /// The derivation --input-scores used to be handed ready-made. Its + /// directory form globbed a directory and preferred the reconciled sibling per stem; + /// this is that rule, applied to the runs the command line names instead of to + /// whatever a directory happened to hold. The difference matters twice: a directory + /// with a stray parquet no longer changes the cohort, and ORDER is now the caller's + /// (FirstPassFDR reconciliation is order-sensitive, so a chain must pass a + /// deterministically sorted list - which is what it already did to get a stable + /// directory sort). + /// + internal static List ScoresPathsForInputs(OspreyConfig config) + { + var paths = new List(config.InputFiles?.Count ?? 0); + if (config.InputFiles == null) + return paths; + foreach (string input in config.InputFiles) + { + paths.Add(ParquetScoreCache.EffectiveScoresPathFromScoresPath( + ParquetScoreCache.GetScoresPath(input))); + } + return paths; + } + /// /// True when the --task SecondPassFDR merge may hand Stage 7 a per-run source /// instead of every run's survivors at once. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index f9d9534539..89e80ce17c 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -58,14 +58,10 @@ internal sealed class SecondPassFdrTask : OspreyTask public override bool IsIncluded(PipelineContext ctx) { var c = ctx.Config; - bool inputs = c.InputScores != null && c.InputScores.Count > 0; - // StopAfterStage5 on BOTH input routes, for the reason PerFileRescoreTask.IsIncluded - // states: it appeared only in the --input-scores clause because --task FirstPassFDR - // was the only setter and that task rejects -i. --task ModelDiagnostics sets it too - // and takes -i. - return (!inputs && !c.NoJoin && !c.StopAfterStage5) - || (inputs && c.ExpectReconciledInput) - || (inputs && !c.NoJoin && !c.StopAfterStage5 && !c.ExpectReconciledInput); + // Its own node always, and the full pipeline unless something stops earlier. + // StopAfterStage5 means that boundary whatever the inputs look like, for the + // reason PerFileRescoreTask.IsIncluded states. + return c.ExpectReconciledInput || (!c.NoJoin && !c.StopAfterStage5); } // Phase B resume surface. Reads each file's reconciled diff --git a/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs b/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs index f203376c0f..3d5006d548 100644 --- a/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs @@ -175,11 +175,11 @@ private static void ValidateIdentityFieldsSurvive() private static void ValidateEveryLegThatHoldsTheLibraryReleasesIt() { AssertRunsOnLeg(true, @"straight-through", new OspreyConfig()); - AssertRunsOnLeg(true, @"--task SecondPassFDR", - WithInputScores(c => c.ExpectReconciledInput = true)); - AssertRunsOnLeg(true, @"--input-scores full pipeline", WithInputScores(_ => { })); - AssertRunsOnLeg(false, @"--task FirstPassFDR", - WithInputScores(c => c.StopAfterStage5 = true)); + AssertRunsOnLeg(true, @"--task SecondPassFDR", ForTask(HpcTask.SecondPassFdr)); + // The `--input-scores full pipeline` leg that stood here is gone with the flag: + // a single-node full pipeline started from parquets IS the straight-through leg + // above now, asserted once rather than twice under two input kinds. + AssertRunsOnLeg(false, @"--task FirstPassFDR", ForTask(HpcTask.FirstPassFdr)); // --fdrbench-pass 1 forces the RESIDENT first-pass pool, which never computes a // surviving base_id set, so there is nothing to release against. @@ -226,22 +226,22 @@ private static void ValidateValidityKeySuffixTracksWhetherTheReleaseRan() { AssertSuffix(true, @"straight-through, released", new OspreyConfig()); AssertSuffix(true, @"--task SecondPassFDR, released", - WithInputScores(c => c.ExpectReconciledInput = true)); + ForTask(HpcTask.SecondPassFdr)); AssertSuffix(true, @"--task FirstPassFDR cannot release", - WithInputScores(c => c.StopAfterStage5 = true)); + ForTask(HpcTask.FirstPassFdr)); OspreyEnvironment.UseFdrProjection = false; AssertSuffix(false, @"could have released, Stage 5 went resident instead", new OspreyConfig()); // SecondPassFDR's release is its own and does not ride the Stage 5 path. AssertSuffix(true, @"--task SecondPassFDR ignores OSPREY_FDR_PROJECTION", - WithInputScores(c => c.ExpectReconciledInput = true)); + ForTask(HpcTask.SecondPassFdr)); OspreyEnvironment.UseFdrProjection = savedProjection; OspreyEnvironment.ReleaseLibraryFragments = false; AssertSuffix(false, @"opted out where a release was possible", new OspreyConfig()); AssertSuffix(true, @"opted out where it was not possible anyway", - WithInputScores(c => c.StopAfterStage5 = true)); + ForTask(HpcTask.FirstPassFdr)); } finally { @@ -270,11 +270,21 @@ private static PipelineContext MakeContext(OspreyConfig config) return new PipelineContext(config, AnalysisPipeline.CanonicalPipeline(), null, null, null); } - private static OspreyConfig WithInputScores(Action set) + /// + /// One task's config, built the way Program.Main builds it: the task, and the + /// three membership flags DERIVED from it. It used to carry an input KIND as well - a + /// parquet list standing for --input-scores - which the release predicate read + /// alongside the flags; that seam has retired. + /// + private static OspreyConfig ForTask(HpcTask task) { - var config = new OspreyConfig { InputScores = new List { @"a.scores.parquet" } }; - set(config); - return config; + return new OspreyConfig + { + SelectedTask = task, + NoJoin = task == HpcTask.PerFileScoring || task == HpcTask.PerFileRescore, + StopAfterStage5 = task == HpcTask.FirstPassFdr || task == HpcTask.ModelDiagnostics, + ExpectReconciledInput = task == HpcTask.SecondPassFdr, + }; } /// diff --git a/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs b/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs index 4f25ef9524..000457c1c9 100644 --- a/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs @@ -21,8 +21,6 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; using Microsoft.VisualStudio.TestTools.UnitTesting; using pwiz.Osprey.Core; using pwiz.Osprey.Tasks; @@ -45,11 +43,22 @@ namespace pwiz.Osprey.Test [TestClass] public class PipelineMembershipTest { - private static OspreyConfig WithInputScores(Action set) + /// + /// One task's config, built the way Program.Main builds it: the task, and the + /// three membership flags DERIVED from it. Nothing else - which is the change these + /// rows record. Each row used to carry an input KIND too (a parquet list standing for + /// --input-scores), and every predicate read both; the kind is gone and the + /// expected memberships below are unchanged, which is the claim worth pinning. + /// + private static OspreyConfig ForTask(HpcTask task) { - var config = new OspreyConfig { InputScores = new List { @"a.scores.parquet" } }; - set(config); - return config; + return new OspreyConfig + { + SelectedTask = task, + NoJoin = task == HpcTask.PerFileScoring || task == HpcTask.PerFileRescore, + StopAfterStage5 = task == HpcTask.FirstPassFdr || task == HpcTask.ModelDiagnostics, + ExpectReconciledInput = task == HpcTask.SecondPassFdr, + }; } [TestMethod] @@ -61,20 +70,24 @@ public void TestIsIncludedMembershipTable() { (@"straight-through", new OspreyConfig(), new[] { true, true, true, true }), - (@"PerFileScoring", new OspreyConfig { NoJoin = true }, + (@"PerFileScoring", ForTask(HpcTask.PerFileScoring), new[] { true, false, false, false }), - (@"FirstPassFDR", WithInputScores(c => c.StopAfterStage5 = true), + (@"FirstPassFDR", ForTask(HpcTask.FirstPassFdr), new[] { false, true, false, false }), - (@"PerFileRescoring", WithInputScores(c => c.NoJoin = true), + (@"PerFileRescoring", ForTask(HpcTask.PerFileRescore), new[] { false, false, true, false }), - (@"SecondPassFDR", WithInputScores(c => c.ExpectReconciledInput = true), + (@"SecondPassFDR", ForTask(HpcTask.SecondPassFdr), new[] { false, false, false, true }), - // --input-scores with no --task: the single-node full pipeline. - // PerFileScoring lazy-rehydrates the supplied scores rather than - // computing them, so it is excluded; FirstPassFDR..SecondPassFDR compute - // Stages 5-8. - (@"input-scores-full", WithInputScores(_ => { }), - new[] { false, true, true, true }), + // --task ModelDiagnostics is a RENDER over retained products, not a stage. + // It needs the per-file load (so PerFileScoring is in) and first-pass state + // (so FirstPassFDR is), and nothing after: a diagnostics fold publishes + // neither CompactedEntries nor a second pass, and the two tasks that demand + // them used to join anyway and fail the run AFTER writing the report it was + // asked for. The row that stood here was `input-scores-full` - the + // single-node full pipeline started from parquets - and it retired with the + // flag; this is the mode that was actually at risk. + (@"ModelDiagnostics", ForTask(HpcTask.ModelDiagnostics), + new[] { true, true, false, false }), }; foreach (var c in cases) diff --git a/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs b/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs index 499b4a069b..8aaefa932f 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs @@ -34,8 +34,9 @@ namespace pwiz.Osprey.Test { /// /// Tests for Osprey Program-level helpers: the HPC scoring split - /// flag validation (Program.ValidateArgs) and the --input-scores - /// directory expansion (Program.ResolveInputScores). + /// flag validation (Program.ValidateArgs). The --input-scores directory + /// expansion it also covered went with that flag: every task takes the data + /// files and derives its parquets from their stems. /// /// These are unit tests of CLI argument plumbing only. End-to-end /// scoring round-trip (Stages 1-4 → parquet → Stage 5+) is exercised @@ -67,7 +68,7 @@ public void RestoreExperimentAgg() OspreyEnvironment.MeanBestN = _savedMeanBestN; } - // --- ValidateArgs: --task is authoritative over input type -------- + // --- ValidateArgs: what each task requires ------------------------- private static OspreyConfig TaskConfig(HpcTask task) { @@ -103,8 +104,6 @@ public void TestValidateSpectraCache() Assert.IsNull(Program.ValidateArgs(config), "a library should be tolerated"); AssertSpectraCacheError(c => { }, "--input c.InputScores = new List { "a.scores.parquet" }, - "not --input-scores"); } private static void AssertSpectraCacheError(Action mutate, string expected) @@ -150,35 +149,20 @@ public void TestValidatePerFileScoringRequiresLibrary() StringAssert.Contains(err, "--library"); } - [TestMethod] - public void TestValidatePerFileScoringRejectsInputScores() - { - // --task is authoritative: PerFileScoring + --input-scores must - // error, not silently dispatch PerFileRescore. - var config = TaskConfig(HpcTask.PerFileScoring); - config.InputScores = new List { "a.scores.parquet" }; - config.LibrarySource = LibrarySource.FromPath("ref.blib"); - config.OutputBlib = "out.blib"; - string err = Program.ValidateArgs(config); - Assert.IsNotNull(err); - StringAssert.Contains(err, "--task PerFileScoring"); - StringAssert.Contains(err, "not --input-scores"); - } - - // - PerFileRescore (--input-scores in) -- + // - PerFileRescore (one run in, its reconciled parquet out) -- [TestMethod] public void TestValidatePerFileRescoreHappyPath() { var config = TaskConfig(HpcTask.PerFileRescore); - config.InputScores = new List { "a.scores.parquet" }; + config.InputFiles = new List { "a.mzML" }; config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; Assert.IsNull(Program.ValidateArgs(config)); } [TestMethod] - public void TestValidatePerFileRescoreRequiresInputScores() + public void TestValidatePerFileRescoreRequiresInput() { var config = TaskConfig(HpcTask.PerFileRescore); config.LibrarySource = LibrarySource.FromPath("ref.blib"); @@ -186,78 +170,49 @@ public void TestValidatePerFileRescoreRequiresInputScores() string err = Program.ValidateArgs(config); Assert.IsNotNull(err); StringAssert.Contains(err, "--task PerFileRescoring"); - StringAssert.Contains(err, "--input-scores"); + StringAssert.Contains(err, "--input"); } [TestMethod] public void TestValidatePerFileRescoreRequiresLibraryAndOutput() { - var config = TaskConfig(HpcTask.PerFileRescore); - config.InputScores = new List { "a.scores.parquet" }; - string err = Program.ValidateArgs(config); - Assert.IsNotNull(err); - StringAssert.Contains(err, "--task PerFileRescoring"); - StringAssert.Contains(err, "--library and --output"); - } - - [TestMethod] - public void TestValidatePerFileRescoreRejectsInputMzml() - { - // Authoritative: PerFileRescore + -i mzML must error, not silently - // dispatch PerFileScoring. Error must name the task the user typed. var config = TaskConfig(HpcTask.PerFileRescore); config.InputFiles = new List { "a.mzML" }; - config.LibrarySource = LibrarySource.FromPath("ref.blib"); - config.OutputBlib = "out.blib"; string err = Program.ValidateArgs(config); Assert.IsNotNull(err); StringAssert.Contains(err, "--task PerFileRescoring"); - StringAssert.Contains(err, "not -i "); + StringAssert.Contains(err, "--library and --output"); } - // - FirstPassFDR (--input-scores in, 2+ files, reconciliation on) -- + // - FirstPassFDR (2+ runs in, reconciliation on) -- [TestMethod] public void TestValidateFirstPassFdrHappyPath() { var config = TaskConfig(HpcTask.FirstPassFdr); - config.InputScores = new List { "a.scores.parquet", "b.scores.parquet" }; + config.InputFiles = new List { "a.mzML", "b.mzML" }; config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; Assert.IsNull(Program.ValidateArgs(config)); } [TestMethod] - public void TestValidateFirstPassFdrRequiresInputScores() - { - var config = TaskConfig(HpcTask.FirstPassFdr); - config.LibrarySource = LibrarySource.FromPath("ref.blib"); - config.OutputBlib = "out.blib"; - string err = Program.ValidateArgs(config); - Assert.IsNotNull(err); - StringAssert.Contains(err, "--task FirstPassFDR"); - StringAssert.Contains(err, "--input-scores"); - } - - [TestMethod] - public void TestValidateFirstPassFdrRejectsInputMzml() + public void TestValidateFirstPassFdrRequiresInput() { var config = TaskConfig(HpcTask.FirstPassFdr); - config.InputFiles = new List { "a.mzML" }; - config.InputScores = new List { "a.scores.parquet", "b.scores.parquet" }; config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; string err = Program.ValidateArgs(config); Assert.IsNotNull(err); StringAssert.Contains(err, "--task FirstPassFDR"); - StringAssert.Contains(err, "cannot be combined with --input"); + StringAssert.Contains(err, "--input"); } [TestMethod] public void TestValidateFirstPassFdrRequiresLibraryAndOutput() { var config = TaskConfig(HpcTask.FirstPassFdr); - config.InputScores = new List { "a.scores.parquet", "b.scores.parquet" }; + config.InputFiles = new List { "a.mzML", "b.mzML" }; string err = Program.ValidateArgs(config); Assert.IsNotNull(err); StringAssert.Contains(err, "--task FirstPassFDR"); @@ -270,20 +225,20 @@ public void TestValidateFirstPassFdrRejectsSingleFile() // FirstPassFDR writes the Stage 5 -> Stage 6 boundary pair, only // meaningful with siblings; a single-file run errors fast. var config = TaskConfig(HpcTask.FirstPassFdr); - config.InputScores = new List { "only.scores.parquet" }; + config.InputFiles = new List { "only.mzML" }; config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; string err = Program.ValidateArgs(config); Assert.IsNotNull(err); StringAssert.Contains(err, "--task FirstPassFDR"); - StringAssert.Contains(err, "2+ parquet files"); + StringAssert.Contains(err, "2+ files"); } [TestMethod] public void TestValidateFirstPassFdrRequiresReconciliationEnabled() { var config = TaskConfig(HpcTask.FirstPassFdr); - config.InputScores = new List { "a.scores.parquet", "b.scores.parquet" }; + config.InputFiles = new List { "a.mzML", "b.mzML" }; config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; config.Reconciliation.Enabled = false; @@ -292,69 +247,42 @@ public void TestValidateFirstPassFdrRequiresReconciliationEnabled() StringAssert.Contains(err, "Reconciliation.Enabled"); } - // - SecondPassFDR (reconciled --input-scores in) -- + // - SecondPassFDR (every run in, reading their reconciled parquets) -- [TestMethod] public void TestValidateSecondPassFdrHappyPath() { - var config = TaskConfig(HpcTask.SecondPassFdr); - config.InputScores = new List { "a.scores-reconciled.parquet" }; - config.LibrarySource = LibrarySource.FromPath("ref.blib"); - config.OutputBlib = "out.blib"; - Assert.IsNull(Program.ValidateArgs(config)); - } - - [TestMethod] - public void TestValidateSecondPassFdrRequiresInputScores() - { - // Uncontested gap from ultrareview: --task SecondPassFDR without - // --input-scores (even with -i mzML) used to pass validation and - // silently run the full pipeline. It must now fail fast. var config = TaskConfig(HpcTask.SecondPassFdr); config.InputFiles = new List { "a.mzML" }; config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; - string err = Program.ValidateArgs(config); - Assert.IsNotNull(err); - StringAssert.Contains(err, "--task SecondPassFDR"); - // -i present -> the cross is reported first; either way it must not pass. + Assert.IsNull(Program.ValidateArgs(config)); } [TestMethod] - public void TestValidateSecondPassFdrRequiresInputScoresNoMzml() + public void TestValidateSecondPassFdrRequiresInput() { + // Uncontested gap from ultrareview: --task SecondPassFDR with no inputs at + // all used to pass validation and silently run the full pipeline. It must + // fail fast, and the message must name the task the user typed. var config = TaskConfig(HpcTask.SecondPassFdr); config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; string err = Program.ValidateArgs(config); Assert.IsNotNull(err); StringAssert.Contains(err, "--task SecondPassFDR"); - StringAssert.Contains(err, "--input-scores"); + StringAssert.Contains(err, "--input"); } [TestMethod] public void TestValidateSecondPassFdrRequiresLibraryAndOutput() - { - var config = TaskConfig(HpcTask.SecondPassFdr); - config.InputScores = new List { "a.scores-reconciled.parquet" }; - string err = Program.ValidateArgs(config); - Assert.IsNotNull(err); - StringAssert.Contains(err, "--task SecondPassFDR"); - StringAssert.Contains(err, "--library and --output"); - } - - [TestMethod] - public void TestValidateSecondPassFdrRejectsInputMzml() { var config = TaskConfig(HpcTask.SecondPassFdr); config.InputFiles = new List { "a.mzML" }; - config.InputScores = new List { "a.scores-reconciled.parquet" }; - config.LibrarySource = LibrarySource.FromPath("ref.blib"); - config.OutputBlib = "out.blib"; string err = Program.ValidateArgs(config); Assert.IsNotNull(err); StringAssert.Contains(err, "--task SecondPassFDR"); - StringAssert.Contains(err, "cannot be combined with --input"); + StringAssert.Contains(err, "--library and --output"); } // - ModelDiagnostics (the completed run's own command line, replayed) -- @@ -368,7 +296,7 @@ public void TestValidateModelDiagnosticsTakesTheFullPipelineArgs() // ModelDiagnostics - so it must validate exactly as that command line does, and // adding a task-specific rule here would reject the invocation it exists to serve. var config = TaskConfig(HpcTask.ModelDiagnostics); - config.InputScores = new List { "a.scores.parquet", "b.scores.parquet" }; + config.InputFiles = new List { "a.mzML", "b.mzML" }; config.LibrarySource = LibrarySource.FromPath("ref.blib"); config.OutputBlib = "out.blib"; Assert.IsNull(Program.ValidateArgs(config)); @@ -389,7 +317,7 @@ public void TestValidateModelDiagnosticsTakesTheFullPipelineArgs() StringAssert.Contains(err, "No input files"); } - // - Default (no --task): full pipeline from -i mzML or --input-scores -- + // - Default (no --task): the full pipeline -- [TestMethod] public void TestValidateDefaultFullHappyPath() @@ -416,54 +344,6 @@ public void TestValidateDefaultRejectsMissingInput() StringAssert.Contains(err, "No input files"); } - [TestMethod] - public void TestValidateFullFromScoresHappyPath() - { - // No --task + --input-scores: the full pipeline started from scores - // (PerFileScoring lazy-rehydrates). A single file is a legal, - // degenerate case. - var config = new OspreyConfig - { - InputScores = new List { "only.scores.parquet" }, - LibrarySource = LibrarySource.FromPath("ref.blib"), - OutputBlib = "out.blib" - }; - Assert.IsNull(Program.ValidateArgs(config)); - } - - [TestMethod] - public void TestValidateFullFromScoresRequiresLibraryAndOutput() - { - // No --task: the error references --input-scores, not a task the - // user never selected. - var config = new OspreyConfig - { - InputScores = new List { "a.scores.parquet", "b.scores.parquet" }, - LibrarySource = LibrarySource.FromPath("ref.blib"), - // missing OutputBlib - }; - string err = Program.ValidateArgs(config); - Assert.IsNotNull(err); - StringAssert.Contains(err, "--input-scores"); - StringAssert.Contains(err, "--library and --output"); - Assert.IsFalse(err.Contains("--task"), "full-from-scores error must not name a --task: " + err); - } - - [TestMethod] - public void TestValidateFullFromScoresRejectsInputMzml() - { - var config = new OspreyConfig - { - InputFiles = new List { "a.mzML" }, - InputScores = new List { "a.scores.parquet" }, - LibrarySource = LibrarySource.FromPath("ref.blib"), - OutputBlib = "out.blib" - }; - string err = Program.ValidateArgs(config); - Assert.IsNotNull(err); - StringAssert.Contains(err, "cannot be combined with --input"); - } - // --- ResolveTask (--task) ----------------------------------------- [TestMethod] @@ -599,7 +479,7 @@ public void TestModelDiagnosticsDeclaresNoOutputs() private static List SecondPassFdrOutputs(HpcTask task) { var config = TaskConfig(task); - config.InputScores = new List { @"a.scores.parquet", @"b.scores.parquet" }; + config.InputFiles = new List { @"a.mzML", @"b.mzML" }; config.LibrarySource = LibrarySource.FromPath(@"ref.blib"); config.OutputBlib = @"out.blib"; var tasks = AnalysisPipeline.CanonicalPipeline(); @@ -701,140 +581,6 @@ public void TestParseArgsRejectsValueFlagsWithoutValue() } } - // --- ResolveInputScores ------------------------------------------- - - [TestMethod] - public void TestResolveExplicitFilesPassThrough() - { - string dir = NewTempDir(); - try - { - string a = Path.Combine(dir, "a.scores.parquet"); - string b = Path.Combine(dir, "b.scores.parquet"); - File.WriteAllText(a, string.Empty); - File.WriteAllText(b, string.Empty); - var resolved = Program.ResolveInputScores(new List { a, b }); - CollectionAssert.AreEqual(new List { a, b }, resolved); - } - finally - { - Directory.Delete(dir, true); - } - } - - [TestMethod] - public void TestResolveExplicitMissingFileErrors() - { - string dir = NewTempDir(); - try - { - string missing = Path.Combine(dir, "does-not-exist.scores.parquet"); - try - { - Program.ResolveInputScores(new List { missing }); - Assert.Fail("Expected ArgumentException for missing file"); - } - catch (ArgumentException ex) - { - StringAssert.Contains(ex.Message, "not found"); - } - } - finally - { - Directory.Delete(dir, true); - } - } - - [TestMethod] - public void TestResolveDirectoryScansAndSorts() - { - string dir = NewTempDir(); - try - { - File.WriteAllText(Path.Combine(dir, "z.scores.parquet"), string.Empty); - File.WriteAllText(Path.Combine(dir, "a.scores.parquet"), string.Empty); - File.WriteAllText(Path.Combine(dir, "m.scores.parquet"), string.Empty); - File.WriteAllText(Path.Combine(dir, "readme.txt"), string.Empty); - var resolved = Program.ResolveInputScores(new List { dir }); - Assert.AreEqual(3, resolved.Count); - Assert.AreEqual("a.scores.parquet", Path.GetFileName(resolved[0])); - Assert.AreEqual("m.scores.parquet", Path.GetFileName(resolved[1])); - Assert.AreEqual("z.scores.parquet", Path.GetFileName(resolved[2])); - } - finally - { - Directory.Delete(dir, true); - } - } - - [TestMethod] - public void TestResolveDirectoryPrefersReconciledPerStem() - { - // The directory holds both Stage 4 .scores.parquet and Stage 6 - // .scores-reconciled.parquet files. For any stem that has both, - // only the reconciled file is returned; never both. - string dir = NewTempDir(); - try - { - File.WriteAllText(Path.Combine(dir, "a.scores.parquet"), string.Empty); - File.WriteAllText(Path.Combine(dir, "a.scores-reconciled.parquet"), string.Empty); - File.WriteAllText(Path.Combine(dir, "b.scores.parquet"), string.Empty); // no reconciled sibling - File.WriteAllText(Path.Combine(dir, "c.scores-reconciled.parquet"), string.Empty); // no original - // An input stem ending in ".reconciled" stays an original (Copilot - // ambiguity regression guard) - its Stage 4 file must be returned - // as an original, not misread as a reconciled output. - File.WriteAllText(Path.Combine(dir, "d.reconciled.scores.parquet"), string.Empty); - var resolved = Program.ResolveInputScores(new List { dir }); - CollectionAssert.AreEqual( - new[] { "a.scores-reconciled.parquet", "b.scores.parquet", - "c.scores-reconciled.parquet", "d.reconciled.scores.parquet" }, - resolved.ConvertAll(Path.GetFileName)); - // The superseded original must not appear. - CollectionAssert.DoesNotContain(resolved.ConvertAll(Path.GetFileName), "a.scores.parquet"); - } - finally - { - Directory.Delete(dir, true); - } - } - - [TestMethod] - public void TestResolveEmptyDirectoryErrors() - { - string dir = NewTempDir(); - try - { - File.WriteAllText(Path.Combine(dir, "not-a-match.txt"), string.Empty); - try - { - Program.ResolveInputScores(new List { dir }); - Assert.Fail("Expected ArgumentException for directory with no parquets"); - } - catch (ArgumentException ex) - { - StringAssert.Contains(ex.Message, "No *.scores.parquet"); - } - } - finally - { - Directory.Delete(dir, true); - } - } - - [TestMethod] - public void TestResolveEmptyListErrors() - { - try - { - Program.ResolveInputScores(new List()); - Assert.Fail("Expected ArgumentException for empty list"); - } - catch (ArgumentException ex) - { - StringAssert.Contains(ex.Message, "at least one path"); - } - } - // --- OspreyConfig defaults ---------------------------------------- [TestMethod] @@ -842,7 +588,6 @@ public void TestConfigDefaultsDisableHpcMode() { var cfg = new OspreyConfig(); Assert.IsFalse(cfg.NoJoin, "NoJoin should default to false"); - Assert.IsNull(cfg.InputScores, "InputScores should default to null"); } // --- ParquetScoreCache.CheckParquetMetadata ----------------------- diff --git a/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs b/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs index 71d7d12767..0d98eb3469 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ResidentPoolGuardTest.cs @@ -402,30 +402,24 @@ private static void AssertStage6HandoffGuard() [TestMethod] public void TestFirstPassMembershipAcrossTasks() { - var scores = new[] { "a.scores.parquet" }; - // Straight-through (-i, no --task): FirstPassFDR runs. Assert.IsTrue(FirstPassFdrTask.IsIncludedFor(new OspreyConfig())); // --task PerFileScoring / PerFileRescoring set NoJoin: excluded, they stop before - // the join. + // the join. One row each is enough now: these used to be asserted twice, once + // with a parquet list and once without, because the predicate read the input KIND + // as well as the flags and the two could disagree. Assert.IsFalse(FirstPassFdrTask.IsIncludedFor( new OspreyConfig { NoJoin = true })); - Assert.IsFalse(FirstPassFdrTask.IsIncludedFor( - new OspreyConfig { NoJoin = true, InputScores = scores.ToList() })); // --task FirstPassFDR sets StopAfterStage5: it IS the first-pass node. Assert.IsTrue(FirstPassFdrTask.IsIncludedFor( - new OspreyConfig { StopAfterStage5 = true, InputScores = scores.ToList() })); - - // The full --input-scores pipeline (no --task): runs. - Assert.IsTrue(FirstPassFdrTask.IsIncludedFor( - new OspreyConfig { InputScores = scores.ToList() })); + new OspreyConfig { StopAfterStage5 = true })); // --task SecondPassFDR: NoJoin FALSE, so the old !NoJoin proxy said "runs" - but // ExpectReconciledInput excludes it. This single row is the whole change. Assert.IsFalse(FirstPassFdrTask.IsIncludedFor( - new OspreyConfig { ExpectReconciledInput = true, InputScores = scores.ToList() }), + new OspreyConfig { ExpectReconciledInput = true }), "--task SecondPassFDR must not be treated as running first-pass Percolator"); // And the consequence the loader draws from it: the merge no longer demands the @@ -440,8 +434,7 @@ public void TestFirstPassMembershipAcrossTasks() // near-empty .blib with no error. Streaming hydrate and lean projection are // different routes; only the first is what this row unlocks. Assert.IsFalse(PerFileScoringTask.NeedsResidentPool( - new OspreyConfig { ExpectReconciledInput = true, InputScores = scores.ToList() }, - useFdrProjection: true)); + new OspreyConfig { ExpectReconciledInput = true }, useFdrProjection: true)); } } } diff --git a/pwiz_tools/Osprey/Osprey/AnalysisPipeline.cs b/pwiz_tools/Osprey/Osprey/AnalysisPipeline.cs index 2d2811c9b9..3d233feab5 100644 --- a/pwiz_tools/Osprey/Osprey/AnalysisPipeline.cs +++ b/pwiz_tools/Osprey/Osprey/AnalysisPipeline.cs @@ -61,26 +61,13 @@ public int Run(OspreyConfig config) // OSPREY_DUMP_* / OSPREY_DIAG_* env var is set. OspreyDiagnostics.Initialize(config.Diagnostics); - // Worker-mode entry normalization: in --input-scores modes - // without explicit -i, synthesize InputFiles from the parquet - // stems ONCE here, at pipeline entry, so the driver's - // Outputs/IsTaskAlreadyDone skip checks and every per-task - // accessor see a populated InputFiles regardless of which task - // the run starts at. (Mutation-contract: InputFiles is a - // pipeline-populated field that does NOT feed any identity - // hash, so it may be written once at entry -- see - // PipelineContext.Config. Previously this lived inside - // PerFileScoringTask's join-only load, which the driver never - // reached when PerFileScoring was the StartAt task, e.g. - // `--task PerFileScoring --input-scores`.) - if (config.InputScores != null && config.InputScores.Count > 0 - && (config.InputFiles == null || config.InputFiles.Count == 0)) - { - var synthetic = new List(config.InputScores.Count); - foreach (var p in config.InputScores) - synthetic.Add(RescoreHydration.SyntheticInputFromParquet(p)); - config.InputFiles = synthetic; - } + // No worker-mode entry normalization any more, and its absence is the + // point. A --input-scores run arrived here with parquet paths and no + // InputFiles, so the pipeline's FIRST act was to convert them back into + // data-file names - a round trip through a synthetic .mzML that does + // not exist, purely so the sidecar helpers could derive from a stem. Every + // task now receives the stems it needs on -i, which is the direction the + // derivation was always going. // --task SpectraCache stages data rather than analyzing it: it runs // its own one-task pipeline instead of the canonical four. Selecting diff --git a/pwiz_tools/Osprey/Osprey/OspreyCommandArgs.cs b/pwiz_tools/Osprey/Osprey/OspreyCommandArgs.cs index deeda0cc70..234f2851fd 100644 --- a/pwiz_tools/Osprey/Osprey/OspreyCommandArgs.cs +++ b/pwiz_tools/Osprey/Osprey/OspreyCommandArgs.cs @@ -85,7 +85,8 @@ static OspreyCommandArgs() // deeper path tree reaches it sooner. Past it the failure is a CreateProcess error or a // truncated argument list, neither of which says "too many inputs". // - // --input-scores already avoids this by accepting a directory; -i had no equivalent. + // --input-list is the answer, and since --input-scores retired it is the ONLY one: + // that flag used to accept a directory, which is how the HPC tasks avoided the wall. // One path per line, blank lines and #-comments ignored, composable with -i and with // itself (both append, exactly as repeated -i does). public static readonly OspreyArgument ARG_INPUT_LIST = new OspreyArgument(@"input-list", @@ -223,22 +224,15 @@ static OspreyCommandArgs() public static readonly OspreyArgument ARG_TASK = new OspreyArgument(@"task", new[] { @"SpectraCache", @"PerFileScoring", @"FirstPassFDR", @"PerFileRescoring", @"SecondPassFDR", @"ModelDiagnostics" }, (c, p) => true); - public static readonly OspreyArgument ARG_INPUT_SCORES = new OspreyArgument(@"input-scores", - () => @"", (c, p) => true) { Variadic = true, ProcessVariadic = (c, toks) => - { - // Accumulate across repeated --input-scores flags and re-resolve, matching the - // former switch exactly (Rust clap Vec). ResolveInputScores expands a - // single directory and validates explicit paths. - var scorePaths = new List(); - if (c._config.InputScores != null) - scorePaths.AddRange(c._config.InputScores); - scorePaths.AddRange(toks); - c._config.InputScores = Program.ResolveInputScores(scorePaths); - return true; - } }; + // --input-scores is GONE. It named an input KIND - "you handed me parquets" - which is + // how the Rust pipeline said "Stage 1-4 is already done"; the C# port says that with + // --task plus the per-run validity sidecars, and two seams answering one question is + // what let --task ModelDiagnostics join the pipeline and demand state a diagnostics + // fold never publishes. Every task now takes -i and derives its parquets from the + // input stem, which is the direction every other sidecar already derives in. private static readonly ArgumentGroup GROUP_HPC = new ArgumentGroup(() => @"Distributed / HPC", true, - ARG_TASK, ARG_INPUT_SCORES); + ARG_TASK); // --- Performance ------------------------------------------------------------------ // OUTER vs INNER parallelism, kept deliberately separate. --parallel-files is the @@ -337,7 +331,7 @@ public static IEnumerable UsageBlocks new ParaUsageBlock(@"EXAMPLES:"), new ParaUsageBlock(@" osprey -i sample.mzML -l library.tsv -o results.blib"), new ParaUsageBlock(@" osprey -i *.mzML -l library.tsv -o results.blib --resolution hram"), - new ParaUsageBlock(@"HPC SPLIT (one node = one --task): see --task / --input-scores above."), + new ParaUsageBlock(@"HPC SPLIT (one node = one --task): see --task above."), }; } } @@ -805,23 +799,27 @@ private static void AppendUsageHtmlHpcExamples(StringBuilder sb) sb.AppendLine(@"# split 1 - one process per mzML (writes <stem>.scores.parquet, <stem>.calibration.json beside each input)"); sb.AppendLine(@"Osprey --task PerFileScoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01"); sb.AppendLine(); - sb.AppendLine(@"# join 1 - one process over ALL parquets (pass a directory so the order is deterministic)"); - sb.AppendLine(@"Osprey --task FirstPassFDR --input-scores ./scores_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01"); + sb.AppendLine(@"# join 1 - one process over ALL runs (pass a sorted list so the order is deterministic)"); + sb.AppendLine(@"Osprey --task FirstPassFDR --input-list runs.txt -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01"); sb.AppendLine(@"# writes beside each parquet: <stem>.1st-pass.fdr_scores.bin, <stem>.reconciliation.json"); sb.AppendLine(); sb.AppendLine(@"# split 2 - one process per file (parquet + its two sidecars co-located)"); - sb.AppendLine(@"Osprey --task PerFileRescoring --input-scores s1.scores.parquet -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01"); + sb.AppendLine(@"Osprey --task PerFileRescoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01"); sb.AppendLine(@"# writes: <stem>.scores-reconciled.parquet"); sb.AppendLine(); - sb.AppendLine(@"# join 2 - one process over ALL reconciled parquets (writes out.blib)"); - sb.AppendLine(@"Osprey --task SecondPassFDR --input-scores ./reconciled_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01"); + sb.AppendLine(@"# join 2 - one process over ALL runs, reading their reconciled parquets (writes out.blib)"); + sb.AppendLine(@"Osprey --task SecondPassFDR --input-list runs.txt -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01"); sb.AppendLine(@""); - sb.AppendLine(@"

--input-scores takes a directory (globbed and sorted internally) " + - @"or an explicit file list (used in the order given). FirstPassFDR reconciliation is " + - @"order-sensitive, so for FirstPassFDR and SecondPassFDR pass a directory or a deterministically sorted " + - @"list. The rehydration sidecars must travel with their parquet into each worker's " + - @"working directory. Let the scheduler do the fan-out (one file per split process) rather " + - @"than --parallel-files, which is the single-node multi-file mode.

"); + sb.AppendLine(@"

EVERY task takes -i, naming the DATA files - the same names " + + @"the first split was given. A join task derives each run's parquet and sidecars from " + + @"the input stem, so the data file itself need not still exist: what has to be in the " + + @"worker's working directory (or under --output-dir) is that run's " + + @"artifacts. FirstPassFDR reconciliation is order-sensitive, so pass a " + + @"deterministically sorted list - --input-list takes one path per line and " + + @"is what a cohort past a few hundred runs needs, since -i spends the " + + @"command line at O(files). Let the scheduler do the fan-out (one file per split " + + @"process) rather than --parallel-files, which is the single-node " + + @"multi-file mode.

"); } /// @@ -860,7 +858,6 @@ private class OspreyArgUsageProvider : IArgUsageProvider { @"decoy-pairing-manifest", @"FDRBench 5-column pairing manifest (TSV), used with --decoys-in-library" }, { @"write-pin", @"Write PIN files for external tools" }, { @"task", @"HPC: run exactly one pipeline task (one node = one task). Omit for the full pipeline. SpectraCache stages the .spectra.bin caches; ModelDiagnostics regenerates only the --model-diagnostics report for a COMPLETED run, writing no other artifact." }, - { @"input-scores", @"HPC: one or more .scores.parquet files, or a single directory (non-recursive). Mutex with --input." }, { @"parallel-files", @"Input files scored concurrently (OUTER). Absent: one at a time (default). No value: auto from free RAM and cores. : exactly N regardless of RAM/cores. Distinct from --threads." }, { @"threads", @"Per-file main-search threads (INNER; default: all cores), divided across files run concurrently by --parallel-files" }, { @"timestamp", @"Prefix each output line with [yyyy/MM/dd HH:mm:ss]" }, diff --git a/pwiz_tools/Osprey/Osprey/Program.cs b/pwiz_tools/Osprey/Osprey/Program.cs index d165845497..d38d7bd98a 100644 --- a/pwiz_tools/Osprey/Osprey/Program.cs +++ b/pwiz_tools/Osprey/Osprey/Program.cs @@ -22,7 +22,6 @@ */ using System; -using System.Collections.Generic; using System.IO; using pwiz.Common.SystemUtil; using pwiz.Osprey.Core; @@ -76,7 +75,7 @@ static int Main(string[] args) try { // Scan args for the HPC task selector up front so error - // messages fire before --input-scores resolution. A single + // messages name the task before any other argument is parsed. A single // `--task ` runs exactly one pipeline task (HPC: one // node = one task) by setting the (NoJoin, StopAfterStage5, // ExpectReconciledInput) config flags the four tasks' @@ -120,10 +119,10 @@ static int Main(string[] args) OspreyConfig config = ParseArgs(args); // --task selects one pipeline task; derive the membership flags // the tasks' IsIncluded methods read. ExpectReconciledInput also - // arms the strict-reconciled-input gate (every --input-scores - // parquet must carry osprey.reconciled = "true"). Mirrors Rust's - // main.rs wiring. SelectedTask is kept so ValidateArgs can enforce - // the task<->input-type contract and name the typed task. + // arms the strict-reconciled-input gate (every run's reconciled + // parquet must carry osprey.reconciled = "true"). All three are + // derived from --task and from nothing else, which is what let the + // input KIND retire: it was the OTHER seam saying the same thing. config.SelectedTask = selectedTask; // --task ModelDiagnostics IS the request for the report; without the flag the // run would recompute the pass-2 view and write nothing, a silent no-op. @@ -186,56 +185,69 @@ static int Main(string[] args) Directory.CreateDirectory(config.OutputDir); if (!string.IsNullOrEmpty(config.CacheDir)) Directory.CreateDirectory(config.CacheDir); - // Runs that consume --input-scores (FirstPassFDR, PerFileRescore, - // SecondPassFDR, or the default full pipeline started from scores) - // have no mzML inputs to validate and ignore --output handling - // differently from per-file scoring. - bool fromInputScores = config.InputScores != null && config.InputScores.Count > 0; - // --task PerFileScoring ignores --output (it writes per-file // .scores.parquet, not a blib), but that is expected single-task / // HPC-worker behavior -- wrapper scripts routinely pass a placeholder // --output -- so it is NOT warned about. The settings block below // reports the real per-file parquet output for this task instead. - // Validate input files exist on disk (skip when consuming - // --input-scores, where there are no mzML inputs; --input-scores - // paths were already validated by ResolveInputScores during parsing). - if (!fromInputScores) + // Validate input files exist on disk. EVERY run reaches this now: a task + // that starts after Stage 4 used to be handed parquets and skipped the + // check entirely, and it is handed the same data-file names as every other + // task instead. + int cacheOnlyInputs = 0; + int artifactOnlyInputs = 0; + foreach (string inputFile in config.InputFiles) { - int cacheOnlyInputs = 0; - foreach (string inputFile in config.InputFiles) + // A directory counts as present. Several vendor formats ARE + // directories (Agilent .d, Bruker .d, Waters .raw), so testing + // File.Exists alone rejected every one of them here, before any + // reader was consulted, on builds with and without the vendor + // reader. It also blocked reusing a raw-derived .spectra.bin, + // which must work on a build that cannot read the raw itself. + if (File.Exists(inputFile) || Directory.Exists(inputFile)) + continue; + // An absent source is fine once its cache is built: Stage 1 is + // the only stage that reads a source, and SpectraCache already + // treats a missing one as "trust the cache". That makes + // delete-the-sources-after-caching a supported way to halve the + // disk a large cohort needs. + if (File.Exists(SpectraCache.GetCachePath(inputFile))) { - // A directory counts as present. Several vendor formats ARE - // directories (Agilent .d, Bruker .d, Waters .raw), so testing - // File.Exists alone rejected every one of them here, before any - // reader was consulted, on builds with and without the vendor - // reader. It also blocked reusing a raw-derived .spectra.bin, - // which must work on a build that cannot read the raw itself. - if (!File.Exists(inputFile) && !Directory.Exists(inputFile)) - { - // An absent source is fine once its cache is built: Stage 1 is - // the only stage that reads a source, and SpectraCache already - // treats a missing one as "trust the cache". That makes - // delete-the-sources-after-caching a supported way to halve the - // disk a large cohort needs. - if (File.Exists(SpectraCache.GetCachePath(inputFile))) - { - cacheOnlyInputs++; - continue; - } - LogError(string.Format("Input file not found: {0}", inputFile)); - return 1; - } + cacheOnlyInputs++; + continue; } - // Announced, not silent: a run whose sources are gone cannot rebuild a - // cache that turns out to be wrong, so the log is the only provenance. - if (cacheOnlyInputs > 0) + // ...and so is an absent source with no cache, once its SCORES exist. + // A join node is shipped parquets and sidecars and nothing else - that + // is the whole point of the split - so demanding the data file back + // would refuse the configuration the HPC chain is built on. This is + // what --input-scores used to say by naming a different input KIND; + // said here it is one input kind and one question about it. + if (File.Exists(ParquetScoreCache.EffectiveScoresPathFromScoresPath( + ParquetScoreCache.GetScoresPath(inputFile)))) { - LogInfo(string.Format( - "{0} of {1} input(s) are absent but have a spectra cache; reading those from the cache.", - cacheOnlyInputs, config.InputFiles.Count)); + artifactOnlyInputs++; + continue; } + LogError(string.Format( + "Input file not found, and it has neither a spectra cache nor a scores " + + "parquet to stand in for it: {0}", inputFile)); + return 1; + } + // Announced, not silent: a run whose sources are gone cannot rebuild a + // cache that turns out to be wrong, so the log is the only provenance. + if (cacheOnlyInputs > 0) + { + LogInfo(string.Format( + "{0} of {1} input(s) are absent but have a spectra cache; reading those from the cache.", + cacheOnlyInputs, config.InputFiles.Count)); + } + if (artifactOnlyInputs > 0) + { + LogInfo(string.Format( + "{0} of {1} input(s) are absent and have no spectra cache; reading those from " + + "their scores parquet, which is what a task after Stage 4 needs.", + artifactOnlyInputs, config.InputFiles.Count)); } if (config.LibrarySource != null && !File.Exists(config.LibrarySource.Path)) { @@ -262,7 +274,7 @@ static int Main(string[] args) // writes --output.) if (config.SelectedTask == HpcTask.SpectraCache) LogInfo("Output: per-file .spectra.bin (no scoring; --output and --library are not used)"); - else if (config.NoJoin && !fromInputScores) + else if (config.NoJoin && config.SelectedTask == HpcTask.PerFileScoring) LogInfo("Output: per-file .scores.parquet (next to each input file)"); else if (config.DiagnosticsOnly) { @@ -355,7 +367,7 @@ static int Main(string[] args) } // Single entry point. The rescore worker (--task - // PerFileRescore, with --input-scores) includes only + // PerFileRescoring) includes only // PerFileRescoreTask (OspreyTask.IsIncluded); PerFileScoring's // lazy-rehydrate (via ctx.Demand) populates the upstream state // from the boundary files on disk. @@ -518,16 +530,17 @@ private static string TaskCliName(HpcTask task) /// /// Validate the parsed config against the selected /// (or the default full pipeline - /// when none was given). When a --task is selected the task is - /// authoritative: it dictates the input type, and the cross - /// (e.g. --task PerFileScoring --input-scores) is rejected rather - /// than silently dispatching the other task. Returns null on success or - /// an error message string on failure. Does not log warnings (those stay - /// in ). Internal so Osprey.Test can exercise it. + /// when none was given). Every task takes the SAME input kind now - the data + /// files, named with -i or --input-list - so what is validated is + /// presence and count, not kind. The cross this used to reject + /// (--task PerFileScoring --input-scores) cannot be expressed any more, + /// which is the point of retiring the second seam rather than teaching a third + /// predicate about it. Returns null on success or an error message string on + /// failure. Does not log warnings (those stay in ). Internal so + /// Osprey.Test can exercise it. /// internal static string ValidateArgs(OspreyConfig config) { - bool hasInputScores = config.InputScores != null && config.InputScores.Count > 0; bool hasInputFiles = config.InputFiles != null && config.InputFiles.Count > 0; // OSPREY_EXPERIMENT_AGG family, before any I/O. Checked here rather than at the @@ -535,7 +548,7 @@ internal static string ValidateArgs(OspreyConfig config) // large run spends reaching FirstPassFDR, and so a warm resume - which skips // FirstPassFdrTask.Run entirely - is still checked. string aggErr = OspreyEnvironment.ValidateExperimentAggSettings( - ExperimentAggFileCount(config, hasInputScores, hasInputFiles)); + ExperimentAggFileCount(config, hasInputFiles)); if (aggErr != null) return aggErr; @@ -548,20 +561,12 @@ internal static string ValidateArgs(OspreyConfig config) // does NOT require --library: caching depends only on the // input file, and demanding one would make staging a dataset // wait on a library that is often chosen later. - if (hasInputScores) - { - return "--task SpectraCache takes -i , not --input-scores " + - "(it builds spectra caches from raw inputs, not from scores)."; - } if (!hasInputFiles) return "--task SpectraCache requires --input ."; return null; case HpcTask.PerFileScoring: // Stage 1-4 worker: mzML in, per-file .scores.parquet out. - if (hasInputScores) - return "--task PerFileScoring takes -i , not --input-scores " + - "(did you mean --task PerFileRescoring?)."; if (!hasInputFiles) return "--task PerFileScoring requires --input ."; if (config.LibrarySource == null) @@ -569,32 +574,31 @@ internal static string ValidateArgs(OspreyConfig config) return null; case HpcTask.PerFileRescore: - // Stage 6 worker: --input-scores in, reconciled per-file out. - if (hasInputFiles) - return "--task PerFileRescoring takes --input-scores, not -i " + - "(mzML paths are derived from the parquet stems)."; - if (!hasInputScores) - return "--task PerFileRescoring requires --input-scores ."; + // Stage 6 worker: one run's scores in, its reconciled parquet out. + // Named by its DATA file like every other task; the parquet and + // sidecars are derived from the stem, and the data file itself need + // not exist (Main's input check accepts a run whose scores are on + // disk). + if (!hasInputFiles) + return "--task PerFileRescoring requires --input ."; if (config.LibrarySource == null || string.IsNullOrEmpty(config.OutputBlib)) return "--task PerFileRescoring requires --library and --output."; return null; case HpcTask.FirstPassFdr: - if (hasInputFiles) - return "--task FirstPassFDR cannot be combined with --input. Use --input-scores instead."; - if (!hasInputScores) - return "--task FirstPassFDR requires --input-scores ."; + if (!hasInputFiles) + return "--task FirstPassFDR requires --input ."; if (config.LibrarySource == null || string.IsNullOrEmpty(config.OutputBlib)) return "--task FirstPassFDR requires --library and --output."; // FirstPassFDR writes the Stage 5 → Stage 6 boundary file // pair, only meaningful with 2+ siblings to reconcile // against and reconciliation enabled. Reject early. - if (config.InputScores.Count < 2) + if (config.InputFiles.Count < 2) return string.Format( - "--task FirstPassFDR requires --input-scores with 2+ parquet files " + - "(got {0}). The Stage 5 → Stage 6 boundary file pair is only meaningful for " + + "--task FirstPassFDR requires --input with 2+ files " + + "(got {0}). The Stage 5 -> Stage 6 boundary file pair is only meaningful for " + "multi-file fan-back-in.", - config.InputScores.Count); + config.InputFiles.Count); if (!config.Reconciliation.Enabled) return "--task FirstPassFDR requires Reconciliation.Enabled = true " + "(got false from config). The Stage 5 → Stage 6 boundary file pair is " + @@ -602,26 +606,17 @@ internal static string ValidateArgs(OspreyConfig config) return null; case HpcTask.SecondPassFdr: - if (hasInputFiles) - return "--task SecondPassFDR cannot be combined with --input. Use --input-scores instead."; - if (!hasInputScores) - return "--task SecondPassFDR requires --input-scores ."; + if (!hasInputFiles) + return "--task SecondPassFDR requires --input ."; if (config.LibrarySource == null || string.IsNullOrEmpty(config.OutputBlib)) return "--task SecondPassFDR requires --library and --output."; return null; } } - // No --task: the full pipeline, started from either -i mzML or - // --input-scores (PerFileScoring lazy-rehydrates the supplied scores). - if (hasInputScores) - { - if (hasInputFiles) - return "--input-scores cannot be combined with --input. Use one or the other."; - if (config.LibrarySource == null || string.IsNullOrEmpty(config.OutputBlib)) - return "--input-scores requires --library and --output."; - return null; - } + // No --task: the full pipeline. A cold run scores from Stage 1; a resume over a + // directory that already holds each run's artifacts skips to whichever stage is + // outstanding, which the per-task validity sidecars decide - not the input kind. if (!hasInputFiles) return "No input files specified. Use -i [file2.mzML ...]"; if (config.LibrarySource == null) @@ -638,8 +633,7 @@ internal static string ValidateArgs(OspreyConfig config) /// never compute an experiment-wide score, so reporting their input count would refuse /// every worker of a legitimate distributed mean(best-N) run. /// - private static int ExperimentAggFileCount( - OspreyConfig config, bool hasInputScores, bool hasInputFiles) + private static int ExperimentAggFileCount(OspreyConfig config, bool hasInputFiles) { switch (config.SelectedTask) { @@ -648,65 +642,9 @@ private static int ExperimentAggFileCount( case HpcTask.PerFileRescore: return 0; } - if (hasInputScores) - return config.InputScores.Count; return hasInputFiles ? config.InputFiles.Count : 0; } - /// - /// Expand --input-scores arguments: a single directory becomes the - /// non-recursive list of *.scores.parquet files in it; explicit file - /// paths are passed through unchanged. Throws if the directory is - /// empty or any explicit path doesn't exist. - /// - /// Directory mode collects both the Stage 4 *.scores.parquet files - /// and the Stage 6 *.scores-reconciled.parquet siblings, then - /// dedupes per stem: for any stem that has both, only the reconciled file - /// is returned (the authoritative later pass; the --task SecondPassFDR - /// reconciled-input gate expects reconciled parquets). A stem with only an - /// original is returned as-is. The two suffixes are unambiguous, so this - /// never returns both files for one stem (see - /// ). - /// - internal static List ResolveInputScores(List paths) - { - if (paths == null || paths.Count == 0) - throw new ArgumentException("--input-scores requires at least one path."); - - if (paths.Count == 1 && Directory.Exists(paths[0])) - { - string dir = paths[0]; - // Glob *.parquet and classify by suffix in code rather than - // relying on multi-dot search-pattern matching (which differs - // across platforms). Keep only the two known scores suffixes. - var originals = new List(); - var reconciledSet = new HashSet(StringComparer.Ordinal); - foreach (string f in Directory.GetFiles(dir, "*.parquet", SearchOption.TopDirectoryOnly)) - { - if (ParquetScoreCache.IsReconciledScoresPath(f)) - reconciledSet.Add(f); - else if (f.EndsWith(ParquetScoreCache.ScoresParquetSuffix, StringComparison.Ordinal)) - originals.Add(f); - } - if (originals.Count == 0 && reconciledSet.Count == 0) - throw new ArgumentException(string.Format( - "No *.scores.parquet files found in --input-scores directory: {0}", dir)); - var result = new List(reconciledSet); // reconciled: authoritative - foreach (string f in originals) - if (!reconciledSet.Contains(ParquetScoreCache.ReconciledPathFromScoresPath(f))) - result.Add(f); // original with no reconciled sibling - result.Sort(StringComparer.Ordinal); // Array.Sort OK: unique filenames, so the comparator never ties - return result; - } - - foreach (string p in paths) - { - if (!File.Exists(p)) - throw new ArgumentException(string.Format("--input-scores path not found: {0}", p)); - } - return paths; - } - internal static void LogInfo(string message) { OspreyOutput.Out.WriteLine(message); diff --git a/pwiz_tools/Osprey/Osprey/RescoreWorker.cs b/pwiz_tools/Osprey/Osprey/RescoreWorker.cs index ff7a0221df..846dd3f8de 100644 --- a/pwiz_tools/Osprey/Osprey/RescoreWorker.cs +++ b/pwiz_tools/Osprey/Osprey/RescoreWorker.cs @@ -73,15 +73,15 @@ namespace pwiz.Osprey public static class RescoreWorker { /// - /// Run the per-file rescore worker on the boundary files - /// referenced by . + /// Run the per-file rescore worker on the boundary files beside the run named by + /// . /// Returns 0 on success, non-zero on failure. /// public static int Run(OspreyConfig config) { // Phase C: the worker is now an alias for the canonical // pipeline entry. The driver runs only the included tasks - // (OspreyTask.IsIncluded): a NoJoin+InputScores config includes + // (OspreyTask.IsIncluded): a --task PerFileRescoring config includes // PerFileRescoreTask, while PerFileScoringTask's probe-the-disk // joinOnly Rehydrate (reached via ctx.Demand) hydrates the upstream // state (stubs, 1st-pass overlay, reconciliation actions, refined diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index 89f50752a7..46d0186fd2 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -32,7 +32,7 @@ layer, and none repeats another: |---|---|---| | **00** (this doc) | Scope, contract, principles, relay | Which file, whose, when, and who may read it | | [14-intermediate-files](14-intermediate-files.md) | Bytes | Headers, versions, schemas, hashing, invalidation mechanics | -| [15-hpc-scoring-split](15-hpc-scoring-split.md) | Operations | CLI flags, `--input-scores` ordering, orchestration recipes | +| [15-hpc-scoring-split](15-hpc-scoring-split.md) | Operations | CLI flags, how a task names its runs and in what order, orchestration recipes | If you are asking "what does this file's header look like?", you want 14. "How do I launch the third worker?" is 15. "Is this task allowed to read that file?" is here. @@ -150,7 +150,7 @@ as the worked one, because for a long time every step in it *was* a fold and the held the pool. The fragment release, the pass-2 competition, protein parsimony, the experiment-q re-clamp and all three `.blib` gates each reduce to `O(distinct)` and each visits every run - but the stage was **handed** every run's survivors before the first of -them started, by the `--input-scores` merge, so nothing they did could bring the peak down. +them started, by the `--task SecondPassFDR` merge, so nothing they did could bring the peak down. At 446 CHS runs that load reached 68.0 GB and was killed at run 381 with 0.34 GB free, having computed nothing. **A fold does not bound anything unless its SOURCE is per-run too**: the runs are now rebuilt one at a time from their own @@ -852,7 +852,7 @@ regardless, and adding a third path to a hash would narrow it further. A warm resume across builds is a separate matter: the version stamp is compared for exact equality (`YEAR.ORDINAL.BRANCH.DOY`) - **but only where it is checked, which is narrower -than it sounds.** That comparison guards the `--input-scores` parquet load. The +than it sounds.** That comparison guards the per-run parquet load. The `.osprey.task` resume path does not do it: `TaskValiditySidecar.IsValid` compares the `validity_key` only, and the `version` field it records is provenance. No version component is in the base key either. So re-invoking the same straight-through command line the next @@ -981,7 +981,7 @@ are functions of all runs: - `.scores.parquet` for **every** run in the cohort - `.calibration.json` for **every** run -- the library, and `--input-scores` naming the parquets +- the library, and `-i` naming the runs whose parquets it reads `.calibration.json` must travel, which is easy to get wrong because the join reads parquets rather than spectra. It supplies RT calibration and the isolation-scheme windows diff --git a/pwiz_tools/Osprey/docs/11-boundary-overrides.md b/pwiz_tools/Osprey/docs/11-boundary-overrides.md index b2faa42d0f..a87c5ff788 100644 --- a/pwiz_tools/Osprey/docs/11-boundary-overrides.md +++ b/pwiz_tools/Osprey/docs/11-boundary-overrides.md @@ -291,7 +291,7 @@ computed by Stage 6 planning. The flags that affect this stage: | Flag / field | Default | Effect on this stage | |--------------|---------|----------------------| | `--task {PerFileScoring\|FirstPassFDR\|PerFileRescoring\|SecondPassFDR}` | (in-process, all stages) | `PerFileRescoring` runs this stage as a standalone worker (internal `HpcTask.PerFileRescore`). `SecondPassFDR` (`HpcTask.SecondPassFdr`) rehydrates reconciled parquets instead of re-scoring. | -| `--input-scores ` | — | Supplies the boundary `.scores.parquet` files the worker rescores; drives `IsIncluded` (`PerFileRescoreTask.cs:123`). | +| `-i ` | — | Names the run the worker rescores; its boundary `.scores.parquet` and sidecars derive from the stem. Membership is `--task` alone (`PerFileRescoreTask.IsIncluded`). | | `--reconciliation-compaction-fdr ` | 0.01 (`OspreyConfig.ReconciliationCompactionFdr`) | First-pass compaction predicate applied upstream in FirstPassFDR; determines which entries survive into the rescore set. | | `ReconciliationConfig.Enabled` | true | Gates reconciliation planning + `reconciliation.json` inputs (`PerFileRescoreTask.cs:158`). Disabling leaves only multi-charge consensus rescore. | | `ReconciliationConfig.ConsensusFdr` | 0.01 (`ReconciliationConfig.cs:39`) | Threshold for consensus peptide selection, calibration refit, and reconciliation planning (`Stage6Planner.cs`). Not a CLI flag; config field. | diff --git a/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md b/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md index 25edc1d2cf..24e533a048 100644 --- a/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md +++ b/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md @@ -6,7 +6,7 @@ For large experiments (hundreds to thousands of mzML files) the Osprey pipeline The C# port implements this split as **four pipeline tasks** driven by a single `--task ` CLI selector, rather than the Rust doc's `--no-join` / `--join-at-pass` / `--join-only` flag family. Each task is a subclass of `OspreyTask` (`Osprey.Tasks/OspreyTask.cs`), and the orchestration model is a per-task membership predicate walked by a driver loop (`Osprey/AnalysisPipeline.cs:99-112`) rather than a contiguous `[start..stop]` stage window. -> **This document owns operations**: CLI flags, the membership truth table, `--input-scores` resolution and ordering, footer-hash validation, and concurrency. *Why* the split has this shape - the scope of each artifact, which task may read what, and the exact file list a node must be shipped at each boundary - is owned by [00-pipeline-architecture.md](00-pipeline-architecture.md), and the byte formats by [14-intermediate-files.md](14-intermediate-files.md). Read 00 before changing what any task writes. +> **This document owns operations**: CLI flags, the membership truth table, how a task names its runs and in what order, footer-hash validation, and concurrency. *Why* the split has this shape - the scope of each artifact, which task may read what, and the exact file list a node must be shipped at each boundary - is owned by [00-pipeline-architecture.md](00-pipeline-architecture.md), and the byte formats by [14-intermediate-files.md](14-intermediate-files.md). Read 00 before changing what any task writes. > **In flight** - this document describes `--task PerFileRescoring` as rehydrating `FirstPassFdrTask` and reading an all-runs `CompactedEntries` buffer (the membership truth table below, and the Stage 6 section). That is what the branch `Skyline/work/20260901_osprey_firstpass_resume` replaces with a per-run hydrate, so both statements change when it lands. **Deviations from the target architecture are tracked in one place - 00's `## In flight` section - not per document**; this note exists so a reader of 15 alone knows to look there. @@ -53,9 +53,9 @@ The exact per-task membership per mode is pinned by `Osprey.Test/PipelineMembers | straight-through (no `--task`, `-i mzML`) | run | run | run | run | | `--task PerFileScoring` (`NoJoin`) | run | – | – | – | | `--task FirstPassFDR` (`StopAfterStage5`) | rehydrate | run | – | – | -| `--task PerFileRescoring` (`NoJoin`+`InputScores`) | rehydrate | rehydrate | run | – | +| `--task PerFileRescoring` (`NoJoin`, `SelectedTask`) | rehydrate | rehydrate | run | – | | `--task SecondPassFDR` (`ExpectReconciledInput`) | rehydrate | (skipped) | rehydrate | run | -| `--input-scores`, no `--task` (single-node full) | rehydrate | run | run | run | +| `--task ModelDiagnostics` (`StopAfterStage5`) | rehydrate | rehydrate | – | – | ("rehydrate" = excluded from the driver loop but lazily materialized on demand from disk; "–" = never touched.) The predicates live in `PerFileScoringTask.IsIncluded` (`:84-88`), `FirstPassFdrTask.IsIncluded` (`:80-95`), `PerFileRescoreTask.IsIncluded` (`:123-130`), and `SecondPassFdrTask.IsIncluded` (`:57-64`). @@ -67,9 +67,9 @@ The exact per-task membership per mode is pinned by `Osprey.Test/PipelineMembers - `.calibration.json` — RT + MS1/MS2 mass calibration (`CalibrationIO.CalibrationPathForInput`). - `.spectra.bin` — the decoded-spectrum cache. No *join* reads it, but Stage 6 rescore does, and it is what lets a search run at all once the input has been deleted (see 14-intermediate-files.md). -The parquet footer is stamped once against the unmutated outer config (`:226-232`) with `osprey.version`, `osprey.search_hash`, `osprey.library_hash`, and `osprey.reconciled = "false"`. Under `--task PerFileScoring` (`config.NoJoin` with no `--input-scores`) the task stops after writing the parquets and returns false with `ExitCode = 0` (`FinalizeAndCheck`, `:649-658`) — Stage 5+ is skipped, no blib is written. `--output` is accepted but not used (`Osprey/Program.cs:228-232` reports the real per-file parquet output instead of warning). +The parquet footer is stamped once against the unmutated outer config (`:226-232`) with `osprey.version`, `osprey.search_hash`, `osprey.library_hash`, and `osprey.reconciled = "false"`. Under `--task PerFileScoring` (`config.NoJoin`, `SelectedTask == PerFileScoring`) the task stops after writing the parquets and returns false with `ExitCode = 0` (`FinalizeAndCheck`, `:649-658`) — Stage 5+ is skipped, no blib is written. `--output` is accepted but not used (`Osprey/Program.cs:228-232` reports the real per-file parquet output instead of warning). -Under `--task PerFileScoring` the task's `IsIncluded` requires **no** `--input-scores` (`:84-88`); `ValidateArgs` rejects `--task PerFileScoring --input-scores` (`Osprey/Program.cs:357-366`). +Under `--task PerFileScoring` the task's `IsIncluded` is true because the task does not start after Stage 4 (`ScoringTaskShared.StartsAfterPerFileScoring`). The cross `ValidateArgs` used to reject here - `--task PerFileScoring --input-scores` - cannot be typed any more, which is the point of retiring the second seam rather than teaching a third predicate about it. `ProcessFile` always writes the parquet regardless of task, matching Rust's end-to-end behavior (the sidecar is needed by Stage 6 reconciliation to lazy-load CWT candidates). @@ -86,7 +86,7 @@ Under `--task PerFileScoring` the task's `IsIncluded` requires **no** `--input-s The boundary file pair per file is thus `.1st-pass.fdr_scores.bin` + `.reconciliation.json`. Each reconciliation.json carries `search_hash`, `library_hash`, the sorted join-wide file-stem set, and the global first-pass passing base_id set (`:970-996`), so a single-file Stage 6 worker can reconstruct the join-wide compaction set. -Under `--task FirstPassFDR` (`config.StopAfterStage5`), `PlanStage6` writes the boundary pair and returns true with `ExitCode = 0` before Stage 6 rescore (`:775-797`). `IsIncluded` requires `--input-scores` with 2+ parquets (`ValidateArgs`, `Osprey/Program.cs:379-399`) and `Reconciliation.Enabled = true`. +Under `--task FirstPassFDR` (`config.StopAfterStage5`), `PlanStage6` writes the boundary pair and returns true with `ExitCode = 0` before Stage 6 rescore (`:775-797`). `ValidateArgs` requires `--input` with 2+ runs and `Reconciliation.Enabled = true`: the Stage 5 -> Stage 6 boundary pair is only meaningful for multi-file fan-back-in. ## Stage 6 — Per-file rescore (`--task PerFileRescoring`) @@ -96,7 +96,7 @@ Under `--task FirstPassFDR` (`config.StopAfterStage5`), `PlanStage6` writes the Reconciled output goes to a **separate** `.scores-reconciled.parquet` sibling, leaving the Stage 4 `.scores.parquet` intact (`ParquetScoreCache.GetReconciledScoresPath`; `WriteReconciledAndStamp`, `:944-987`). Its footer carries `osprey.reconciled = "true"` plus `osprey.reconciliation_hash` (`Osprey.Tasks/ReconciledParquetWriter.cs:198-205`). This differs from the Rust doc, which says Stage 6 "rewrites each `.scores.parquet`" in place (see Divergences). -Under `--task PerFileRescoring` (`config.NoJoin` + `--input-scores`), `IsIncluded` (`:123-130`) includes only this task; `PerFileScoringTask` and `FirstPassFdrTask` lazy-rehydrate the upstream state from the boundary files via `ctx.Demand`. `RescoreWorker.Run` (`Osprey/RescoreWorker.cs:80-91`) is now a thin alias that just calls `new AnalysisPipeline().Run(config)` — the hand-rolled worker path was collapsed into the canonical driver. `ValidateArgs` forbids `-i` (mzML paths are derived from the parquet stems) and requires `--library` + `--output` (`Osprey/Program.cs:368-377`). +Under `--task PerFileRescoring` (`config.NoJoin`, and `SelectedTask` is what distinguishes it from `PerFileScoring` - the input KIND used to), `IsIncluded` includes only this task; `PerFileScoringTask` and `FirstPassFdrTask` lazy-rehydrate the upstream state from the boundary files via `ctx.Demand`. `RescoreWorker.Run` (`Osprey/RescoreWorker.cs:80-91`) is now a thin alias that just calls `new AnalysisPipeline().Run(config)` — the hand-rolled worker path was collapsed into the canonical driver. `ValidateArgs` requires `--input` (the run this worker rescores, whose parquet and sidecars derive from its stem) plus `--library` + `--output`. ## Stages 7-8 — Second-pass FDR (`--task SecondPassFDR`) @@ -107,25 +107,25 @@ Under `--task PerFileRescoring` (`config.NoJoin` + `--input-scores`), `IsInclude 3. Re-clamp experiment q to best run q (`PercolatorEngine.ClampExperimentQToBestRun`, `:177`). 4. Write the BiblioSpecLite `.blib` (`WriteBlibOutput`, `:299-370`; see 13-blib-output-schema.md). -Under `--task SecondPassFDR` (`config.ExpectReconciledInput`), `Rehydrate` (`:317-455`) hydrates from the reconciled parquets + sidecars **without** materializing `FirstPassFdrTask` (which would wrongly re-run Stage 5 Percolator on the reconciled parquets), applies its own compaction, and lets `SecondPassFdrTask.Run` do 2nd-pass FDR + protein FDR + blib. The strict reconciled-input gate asserts every `--input-scores` parquet carries `osprey.reconciled = "true"` (`ParquetScoreCache.ValidateScoresParquetGroup`, `Osprey.IO/ParquetScoreCache.cs:1292-1305`). `ValidateArgs` forbids `-i` and requires `--library` + `--output` (`Osprey/Program.cs:401-408`). +Under `--task SecondPassFDR` (`config.ExpectReconciledInput`), `Rehydrate` (`:317-455`) hydrates from the reconciled parquets + sidecars **without** materializing `FirstPassFdrTask` (which would wrongly re-run Stage 5 Percolator on the reconciled parquets), applies its own compaction, and lets `SecondPassFdrTask.Run` do 2nd-pass FDR + protein FDR + blib. The strict reconciled-input gate asserts every run's reconciled parquet carries `osprey.reconciled = "true"` (`ParquetScoreCache.ValidateScoresParquetGroup`). `ValidateArgs` requires `--input` plus `--library` + `--output`. `SecondPassFdrTask.Rehydrate` returns `true` as a no-op (`:113`): nothing consumes SecondPassFDR's state in-memory, so it is never demanded. -## Full pipeline (default) and `--input-scores` full run +## Full pipeline (default) -With no `--task`, all four tasks run in one process (`straight-through` row of the truth table): output is identical to running the four workers in sequence over the same files. `--input-scores` with no `--task` (the `input-scores-full` row) runs Stages 5-8 in one process from existing per-file parquets — `PerFileScoringTask` is excluded (`IsIncluded` returns false when `InputScores` is non-empty, `:84-88`) and lazy-rehydrates the supplied scores instead of recomputing them. +With no `--task`, all four tasks run in one process (`straight-through` row of the truth table): output is identical to running the four workers in sequence over the same files. A run whose per-file artifacts are already on disk resumes to whichever stage is outstanding - the per-task validity sidecars decide that, not the input kind. -## `--input-scores` resolution and ordering +The row that used to sit beside it, `--input-scores` with no `--task` (a single-node full run started from parquets), retired with the flag. It was the same run: the pipeline resumes from whatever is current in the output directory either way. -`Program.ResolveInputScores` (`Osprey/Program.cs:446-483`), wired through `ARG_INPUT_SCORES` (`Osprey/OspreyCommandArgs.cs:208-220`): +## How a task names its runs, and ordering -- A single **directory** argument is globbed **non-recursively** for `*.parquet`, then classified by suffix: `*.scores-reconciled.parquet` vs `*.scores.parquet` (`:459-465`). -- **Reconciled wins per stem**: a stem with both files returns only the reconciled parquet (the authoritative later pass); a stem with only the original returns the original (`:469-472`). -- The resulting list is **sorted Ordinal** (`:473`), so the file order is deterministic and stable across nodes. -- Explicit path lists are passed through unchanged after existence validation (`:477-482`); repeated `--input-scores` flags accumulate and re-resolve (`OspreyCommandArgs.cs:211-218`). -- Empty directory or missing explicit path throws (`:466-468`, `:479-480`). +Every task takes `-i` / `--input-list`, naming the **data files**, and derives each run's parquet and sidecars from the input stem plus `--output-dir`. The data file need not still exist: `Program.Main`'s input check accepts a run whose `.spectra.bin` is on disk (delete-the-sources-after-caching), and also one whose `.scores.parquet` - or its reconciled sibling - is on disk, which is the state a staged worker directory is in. -At pipeline entry, `AnalysisPipeline.Run` synthesizes `config.InputFiles` from the parquet stems once (`:76-83`, via `RescoreHydration.SyntheticInputFromParquet`) so every per-task accessor sees a populated `InputFiles` regardless of which task the run starts at. +- **Reconciled wins per stem**, as it always did: `ScoringTaskShared.ScoresPathsForInputs` resolves each input through `ParquetScoreCache.EffectiveScoresPathFromScoresPath`, so a run whose Stage 6 output exists is read from the reconciled parquet and one without it from the Stage 4 file. +- **Order is the caller's.** FirstPassFDR reconciliation is order-sensitive, so a chain must pass a deterministically sorted list. `--input-scores` used to sort a globbed directory Ordinal on the caller's behalf; naming the runs explicitly means an orchestrator states the order rather than inheriting it from a directory listing, and a stray parquet in that directory can no longer change the cohort. +- `--input-list` takes one path per line (blank lines and `#` comments ignored) and composes with `-i`. It is what a cohort past a few hundred runs needs: 446 `-i` paths measured ~28,600 characters against a 32,767 limit. + +**Why the flag went.** It named an input KIND - "you handed me parquets" - which is how the Rust pipeline said *Stage 1-4 is done*. The C# port says that with `--task` plus the per-run validity sidecars, and two seams answering one question is what let `--task ModelDiagnostics` (which sets `StopAfterStage5`, a C#-era signal, while its inputs were mzML stems, a Rust-era one) join the pipeline and demand state a diagnostics fold never publishes. The clearest evidence it was a round trip: the pipeline's first act was `RescoreHydration.SyntheticInputFromParquet`, rebuilding a synthetic `.mzML` that does not exist, purely so the sidecar path helpers could work. ## Parquet footer hash validation @@ -183,7 +183,6 @@ a corrupt cache a downstream stage must reject. See principle P8 in | Flag / field | Default | Effect on this stage | |---|---|---| | `--task {PerFileScoring\|FirstPassFDR\|PerFileRescoring\|SecondPassFDR}` | none (full pipeline in one process) | Selects one HPC worker; sets `NoJoin` / `StopAfterStage5` / `ExpectReconciledInput` (`Program.cs:126-128`). Resolved case-insensitively (`ResolveTask`). | -| `--input-scores ` | none | One or more `.scores.parquet` files or a single directory (globbed non-recursively, reconciled-wins-per-stem, Ordinal-sorted). Mutually exclusive with `-i/--input`. Consumed by `FirstPassFDR` / `PerFileRescoring` / `SecondPassFDR` and the `--input-scores`-only full run. | | `-i/--input ` | none | Required by `PerFileScoring` and the default full pipeline; forbidden by `FirstPassFDR` / `PerFileRescoring` / `SecondPassFDR`. | | `-l/--library`, `-o/--output` | none | Required by `FirstPassFDR`, `PerFileRescoring`, and `SecondPassFDR`. `--output` is accepted-but-unused by `--task PerFileScoring` (writes per-file parquets, not a blib). | | `--reconciliation-compaction-fdr ` | 0.01 | Peptide-q gate for Stage 5 compaction (`FirstPassFdrTask.cs:689`). | @@ -202,7 +201,7 @@ a corrupt cache a downstream stage must reject. See principle P8 in - **[INTENTIONAL-CSHARP-DESIGN] One name per task, describing the FDR pass** - The CLI name, the `HpcTask` member, the task class, the `[TASK]` log token, and the `.osprey.task` stamp are all one string per task, describing the FDR pass rather than the join topology. Two of them used to describe the topology instead (`FirstJoinTask`/`FirstPassFDR` and `MergeNodeTask`/`SecondPassFDR`), which cost a reader a mapping table and once produced a resume leg that keyed off the class names, matched zero sidecars, and passed green having resumed nothing; issue #4535 renamed them. The residual mapping is `PerFileRescoring` vs `PerFileRescore`, plus the `Fdr`/`FDR` casing that follows this codebase's type convention (`FdrEntry`, `FdrController`) rather than the all-caps `pwiz.Osprey.FDR` namespace. Folding those two in as well would let `ResolveTask` and `TaskCliName` be deleted outright. Evidence: the `HpcTask` enum in `Osprey.Core/OspreyConfig.cs`, `ResolveTask` / `TaskCliName` in `Osprey/Program.cs`. Severity: info. -- **[INTENTIONAL-CSHARP-DESIGN] Stage 6 writes a separate `.scores-reconciled.parquet`, not an in-place rewrite** - Rust doc says Stage 6 "rewrites each `.scores.parquet`" with reconciled scores; C# writes a separate `.scores-reconciled.parquet` sibling and leaves the Stage 4 parquet intact (crash-safety: a partial Stage 6 crash cannot half-rewrite the Stage 4 output). `--input-scores` directory resolution then prefers the reconciled sibling per stem. Evidence: `Osprey.Tasks/PerFileRescoreTask.cs:163-177,944-954`, `Osprey/Program.cs:459-472`. Severity: minor. +- **[INTENTIONAL-CSHARP-DESIGN] Stage 6 writes a separate `.scores-reconciled.parquet`, not an in-place rewrite** - Rust doc says Stage 6 "rewrites each `.scores.parquet`" with reconciled scores; C# writes a separate `.scores-reconciled.parquet` sibling and leaves the Stage 4 parquet intact (crash-safety: a partial Stage 6 crash cannot half-rewrite the Stage 4 output). Each run's effective parquet then prefers the reconciled sibling (`ParquetScoreCache.EffectiveScoresPathFromScoresPath`). Evidence: `Osprey.Tasks/PerFileRescoreTask.cs`, `Osprey.Tasks/ScoringTaskShared.ScoresPathsForInputs`. Severity: minor. - **[INTENTIONAL-CSHARP-DESIGN] Orchestration is membership-predicate + lazy-rehydrate, not a stage window** - Rust doc frames each mode as "run stages X through Y, load the rest from disk"; C# implements a fixed four-task canonical pipeline where each task's `IsIncluded` decides participation and excluded/valid tasks lazy-rehydrate their state on demand through the typed byproduct registry. Behavior/outputs match the Rust modes (pinned by the membership truth table). Evidence: `Osprey/AnalysisPipeline.cs:99-148`, `Osprey.Test/PipelineMembershipTest.cs:55-93`. Severity: info. @@ -212,4 +211,4 @@ a corrupt cache a downstream stage must reject. See principle P8 in - **[STALE-RUST-DOC] Stage 4 parquet footer omits `osprey.reconciliation_hash`** - Rust doc's hash table lists `osprey.reconciliation_hash` as parquet footer metadata generally; in C# the Stage 4 `.scores.parquet` footer carries only `version` / `search_hash` / `library_hash` / `reconciled = "false"`, and `reconciliation_hash` is written **only** on the Stage 6 reconciled parquet. This matches the semantic intent (the hash is meaningful only post-reconciliation) but the field is not present on every parquet. Evidence: `Osprey.Tasks/PerFileScoringTask.cs:226-232` vs `Osprey.Tasks/ReconciledParquetWriter.cs:198-205`. Severity: info. -Verified as matching the Rust doc: the four-phase split (per-file scoring / FirstPassFDR / per-file rescore / SecondPassFDR) and which stages run vs load-from-disk in each mode; the boundary file pair (`.1st-pass.fdr_scores.bin` + `.reconciliation.json`); the SHA-256 footer-hash validation (version / search_hash / library_hash / reconciled) aborting early with a file-named error; the `--task SecondPassFDR` strict `reconciled = "true"` gate; the `--input-scores` directory being scanned non-recursively; the mutual-exclusion validation errors (`Program.ValidateArgs`); the reconciliation.json carrying `search_hash`/`library_hash`; the copy-and-verify safe-write pattern; and the env-var-gated cross-impl bisection dumps. +Verified as matching the Rust doc: the four-phase split (per-file scoring / FirstPassFDR / per-file rescore / SecondPassFDR) and which stages run vs load-from-disk in each mode; the boundary file pair (`.1st-pass.fdr_scores.bin` + `.reconciliation.json`); the SHA-256 footer-hash validation (version / search_hash / library_hash / reconciled) aborting early with a file-named error; the `--task SecondPassFDR` strict `reconciled = "true"` gate; the per-task input requirements (`Program.ValidateArgs`); the reconciliation.json carrying `search_hash`/`library_hash`; the copy-and-verify safe-write pattern; and the env-var-gated cross-impl bisection dumps. diff --git a/pwiz_tools/Osprey/docs/16-determinism.md b/pwiz_tools/Osprey/docs/16-determinism.md index 5f198eb6d9..51f3ed59f1 100644 --- a/pwiz_tools/Osprey/docs/16-determinism.md +++ b/pwiz_tools/Osprey/docs/16-determinism.md @@ -134,8 +134,10 @@ is seeded and deterministic. There are **two**, structurally different: which run survives follows FILE ORDER. A cross-run maximum was commutative and did not. Re-running the same file list in the same order reproduces the same model; re-running it in a different order does not, and file order is not part of the task validity key. - `Program.ResolveInputScores` sorts the single-directory form but preserves caller order - for the explicit multi-path `--input-scores` form. + Order is the CALLER's: `--input-scores` used to sort a globbed directory on the caller's + behalf, and with it retired an orchestrator states the order explicitly (`--input-list` + takes a sorted file). A stray parquet in a directory can no longer change the cohort + either, which is the other half of the same trade. `XorShift64` (`Osprey.ML/LinearSvmClassifier.cs:266`) matches the Rust generator exactly (`x ^= x << 13; x ^= x >> 7; x ^= x << 17`). diff --git a/pwiz_tools/Osprey/docs/19-testing.md b/pwiz_tools/Osprey/docs/19-testing.md index c0c1934206..705f051017 100644 --- a/pwiz_tools/Osprey/docs/19-testing.md +++ b/pwiz_tools/Osprey/docs/19-testing.md @@ -72,8 +72,8 @@ inputs (spectra, library entries, feature vectors) with no external data dependency. Additional C# test files with no direct Rust analog cover port-specific -infrastructure: `ProgramTests.cs` (CLI `--task` argument validation and -`--input-scores` directory expansion — see 15-hpc-scoring-split.md), +infrastructure: `ProgramTests.cs` (CLI `--task` argument validation — see +15-hpc-scoring-split.md), `ByproductContextTest.cs`, `DiagnosticsTest.cs`, `ModelDiagnosticsDataTest.cs` (the `--model-diagnostics` HTML report), `FileSaverTest.cs` (the safe copy-and-verify NAS-write pattern), `DecoyPairingManifestTest.cs` and diff --git a/pwiz_tools/Osprey/docs/20-command-line.md b/pwiz_tools/Osprey/docs/20-command-line.md index f992eaea8c..153e8228eb 100644 --- a/pwiz_tools/Osprey/docs/20-command-line.md +++ b/pwiz_tools/Osprey/docs/20-command-line.md @@ -139,8 +139,7 @@ Defaults and value lists are from `Osprey/OspreyCommandArgs.cs`; the parser acce | Option | Value | Effect | |--------|-------|--------| -| `--task` | `PerFileScoring \| FirstPassFDR \| PerFileRescoring \| SecondPassFDR` | Run exactly one pipeline task (one node = one task). Omit for the whole pipeline. See [15-hpc-scoring-split.md](15-hpc-scoring-split.md). | -| `--input-scores` | `` | One or more `.scores.parquet` files, or a single directory (non-recursive). Mutually exclusive with `--input`. | +| `--task` | `PerFileScoring \| FirstPassFDR \| PerFileRescoring \| SecondPassFDR` | Run exactly one pipeline task (one node = one task). Omit for the whole pipeline. EVERY task takes `-i`/`--input-list` naming the data files; the parquets and sidecars are derived from their stems. See [15-hpc-scoring-split.md](15-hpc-scoring-split.md). | ### Logging @@ -181,21 +180,26 @@ check rejects inputs whose search/library hash does not match. # split 1 — one process per mzML (writes .scores.parquet, .calibration.json beside each input) osprey --task PerFileScoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 -# join 1 — one process over ALL parquets (pass a DIRECTORY so order is deterministic) -osprey --task FirstPassFDR --input-scores ./scores_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 +# join 1 — one process over ALL runs (pass a sorted list so order is deterministic) +osprey --task FirstPassFDR --input-list runs.txt -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 # writes beside each parquet: .1st-pass.fdr_scores.bin, .reconciliation.json # split 2 — one process per file (parquet + its two sidecars co-located) -osprey --task PerFileRescoring --input-scores s1.scores.parquet -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 +osprey --task PerFileRescoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 # writes: .scores-reconciled.parquet -# join 2 — one process over ALL reconciled parquets (writes out.blib) -osprey --task SecondPassFDR --input-scores ./reconciled_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 +# join 2 — one process over ALL runs, reading their reconciled parquets (writes out.blib) +osprey --task SecondPassFDR --input-list runs.txt -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 ``` -- `--input-scores` takes a **directory** (globbed and sorted internally) or an explicit - file list (consumed in the order given). FirstPassFDR reconciliation is order-sensitive, - so for `FirstPassFDR` and `SecondPassFDR` pass a directory or a deterministically sorted list. +- Every task names its runs by their **data files**, and the data file itself need not + still exist: a task after Stage 4 is accepted when the run's `.scores.parquet` (or its + reconciled sibling) is on disk, which is the state a staged worker directory is in. + `--input-scores`, which named parquets instead, has retired - it was a second way of + saying what `--task` already says. +- FirstPassFDR reconciliation is **order-sensitive**, so pass a deterministically sorted + list. `--input-list` (one path per line) is what a cohort past a few hundred runs needs: + 446 `-i` paths is ~87% of the Windows command-line limit. - Rehydration sidecars must travel with their parquet into each worker's working directory. Let the scheduler fan out (one file per split process) rather than `--parallel-files`, which is the single-node multi-file mode. diff --git a/pwiz_tools/Osprey/docs/DIVERGENCES.md b/pwiz_tools/Osprey/docs/DIVERGENCES.md index a26bc0c1b3..6d995bd19a 100644 --- a/pwiz_tools/Osprey/docs/DIVERGENCES.md +++ b/pwiz_tools/Osprey/docs/DIVERGENCES.md @@ -275,7 +275,7 @@ Legend — Classification: **STALE** = STALE-RUST-DOC, **INTENT** = INTENTIONAL- |---|---|---|---|---|---| | INTENT | CLI is `--task `, not `--no-join`/`--join-at-pass`/`--join-only` | Orchestrated by `--join-at-pass` + modifiers | Single `--task {PerFileScoring\|FirstPassFDR\|PerFileRescoring\|SecondPassFDR}`; old flags retired, fail fast | `Program.cs:86-128`; `OspreyCommandArgs.cs:206-207` | major | | INTENT | One name per task, describing the FDR pass | Named by pass/join topology | CLI name, enum member and class are one name per task; residual `PerFileRescoring` vs `PerFileRescore` | `OspreyConfig.cs` (`HpcTask`); `Program.cs` (`ResolveTask`) | info | -| INTENT | Stage 6 separate `.scores-reconciled.parquet` | Rewrites `.scores.parquet` | Separate sibling; `--input-scores` prefers reconciled | `PerFileRescoreTask.cs:163-177,944-954` | minor | +| INTENT | Stage 6 separate `.scores-reconciled.parquet` | Rewrites `.scores.parquet` | Separate sibling; each run's effective parquet prefers reconciled | `PerFileRescoreTask.cs:163-177,944-954` | minor | | INTENT | Membership-predicate + lazy-rehydrate, not stage window | Each mode runs stages X..Y | Fixed 4-task pipeline; `IsIncluded` + typed byproduct registry; pinned by truth table | `AnalysisPipeline.cs:99-148`; `PipelineMembershipTest.cs:55-93` | info | | INTENT | No `--parquet-compression`; ZSTD unconditional | `--parquet-compression snappy` for OspreySharp interop | Writes ZSTD; read auto-dispatches; cross-impl ZSTD/Snappy read compat is follow-up | `ParquetScoreCache.cs:270,462` | minor | | FLAG | `OSPREY_DUMP_PREDICT_RT` declared but disabled | Stage 6 worker bisection dump | `DumpPredictRt` declared; call site commented out (scoring hotspot) → produces nothing | `IOspreyDiagnostics.cs:80`; `PerFileRescoreTask.cs:715-726` | minor | diff --git a/pwiz_tools/Osprey/docs/README.md b/pwiz_tools/Osprey/docs/README.md index 78f798c819..7cc7f94af0 100644 --- a/pwiz_tools/Osprey/docs/README.md +++ b/pwiz_tools/Osprey/docs/README.md @@ -25,7 +25,7 @@ architecture the C# pipeline is built to, rather than porting a Rust source doc. Three documents divide the file-and-orchestration subject, and none repeats another: **00** owns scope, contract, principles and relay (which file, whose, when, who may read it); **14** owns the bytes (headers, versions, schemas, hashing, invalidation mechanics); **15** -owns operations (CLI flags, `--input-scores` ordering, orchestration recipes). +owns operations (CLI flags, how a task names its runs and in what order, orchestration recipes). | # | Doc | What it covers | |---|-----|----------------| @@ -44,7 +44,7 @@ owns operations (CLI flags, `--input-scores` ordering, orchestration recipes). | 12 | [second-pass-fdr](12-second-pass-fdr.md) | Stage-7 second-pass FDR and the frozen-model q-value modes — transfer-compete and protein-compact — selected by `OSPREY_PASS2_QVALUE`. | | 13 | [blib-output-schema](13-blib-output-schema.md) | BiblioSpec SQLite schema plus Osprey extension tables and the nullable `retentionTime` convention for Skyline ID lines. | | 14 | [intermediate-files](14-intermediate-files.md) | On-disk caches / sidecars (calibration JSON, spectra cache, `.scores.parquet`, FDR sidecars), SHA-256 footer hashing, and the tiered memory architecture. | -| 15 | [hpc-scoring-split](15-hpc-scoring-split.md) | The four `--task` workers, their input/output files, `--input-scores` ordering rules, and validity sidecars for HPC / NextFlow orchestration. | +| 15 | [hpc-scoring-split](15-hpc-scoring-split.md) | The four `--task` workers, their input/output files, the ordering rules a chain must observe, and validity sidecars for HPC / NextFlow orchestration. | | 16 | [determinism](16-determinism.md) | Patterns that keep results bit-identical across runs: thread-order independence, float stability, cross-validation fold assignment. | | 17 | [vectorization](17-vectorization.md) | Performance-critical vectorization — the SIMD / BLAS-equivalent paths for XCorr and matrix operations. | | 18 | [peptide-trace](18-peptide-trace.md) | The per-peptide diagnostic dump facility (C# `OSPREY_DUMP_*` / `OSPREY_DIAG_*` in place of the Rust `OSPREY_TRACE_PEPTIDE`). | diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index fea4e2271f..8114636f08 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -1414,7 +1414,7 @@ function Invoke-HpcChain { # staged dir as CWD. $manifestName = if ($Manifest) { Split-Path -Leaf $Manifest } else { $null } $extraArgs = Get-DatasetCliArgs -Spec $Spec -Manifest $manifestName - # Stable, file-order stem list (NOT hashtable key order) so the --input-scores + # Stable, file-order stem list (NOT hashtable key order) so the -i # argument order matches the straight-through's file order deterministically. $stemList = @($Mzmls | ForEach-Object { [IO.Path]::GetFileNameWithoutExtension($_) }) $mzmlByStem = @{} @@ -1488,8 +1488,12 @@ function Invoke-HpcChain { Copy-Item (Join-Path $ph1Dirs[$s] "$s.calibration.json") (Join-Path $ph2 "$s.calibration.json") } Copy-LibraryInto -Library $Library -Dir $ph2 -Manifest $Manifest + # -i names the DATA files, exactly as phase 1 was given them, even though this + # directory holds no data file at all: Osprey accepts an absent input whose scores + # parquet is on disk, which is precisely the state a staged join node is in. That + # tolerance is what --input-scores used to express by naming a different input KIND. $a2 = @('--task', 'FirstPassFDR') - foreach ($s in $stemList) { $a2 += @('--input-scores', "$s.scores.parquet") } + foreach ($s in $stemList) { $a2 += @('-i', "$s.mzML") } $a2 += @('-l', $libName, '-o', 'output.blib', '--resolution', $Resolution, '--protein-fdr', '0.01', '--threads', $Threads.ToString()) $a2 += $extraArgs @@ -1558,7 +1562,7 @@ function Invoke-HpcChain { $ph2diag = Join-Path $ph2 'output.1st-pass.model-diagnostics.json' if (Test-Path $ph2diag) { Copy-Item $ph2diag (Join-Path $ph3 'output.1st-pass.model-diagnostics.json') } Copy-LibraryInto -Library $Library -Dir $ph3 -Manifest $Manifest - $a3 = @('--task', 'PerFileRescoring', '--input-scores', "$s.scores.parquet", + $a3 = @('--task', 'PerFileRescoring', '-i', "$s.mzML", '-l', $libName, '-o', 'output.blib', '--resolution', $Resolution, '--protein-fdr', '0.01', '--threads', $Threads.ToString()) $a3 += $extraArgs @@ -1697,8 +1701,12 @@ function Invoke-HpcChain { # worker dirs are done. foreach ($d in $ph3Dirs.Values) { Remove-Scratch $d } Copy-LibraryInto -Library $Library -Dir $ph4 -Manifest $Manifest + # -i again, and the RECONCILED parquet is what each run resolves to: this directory + # holds only the reconciled sibling, and EffectiveScoresPathFromScoresPath prefers it. + # Naming it explicitly is what --input-scores did; deriving it is what every other + # reader on this leg already did. $a4 = @('--task', 'SecondPassFDR') - foreach ($s in $stemList) { $a4 += @('--input-scores', "$s.scores-reconciled.parquet") } + foreach ($s in $stemList) { $a4 += @('-i', "$s.mzML") } $a4 += @('-l', $libName, '-o', 'output.blib', '--resolution', $Resolution, '--protein-fdr', '0.01', '--threads', $Threads.ToString()) $a4 += $extraArgs @@ -1913,7 +1921,7 @@ foreach ($name in $selected) { # pool, so no leg of this chain arms the guard at all. Keeping the opt-in would be # actively harmful: # it wrapped the whole chain and would mask a genuine guard regression on any - # --input-scores worker (--task PerFileScoring / PerFileRescoring), which is exactly + # per-file worker (--task PerFileScoring / PerFileRescoring), which is exactly # what mode 3 exists to exercise. $chainBlib = Invoke-HpcChain -Mzmls $inputs.Mzmls -Library $inputs.Library ` -Resolution $cfg.Resolution -ChainRoot $chainRoot -Spec $cfg -Manifest $inputs.Manifest ` From 1a120903fea5238541436f20cb7f7ee178731fc7 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 18:39:56 -0700 Subject: [PATCH 26/30] Addressed Copilot review feedback on PR #4646 * Corrected the PipelineContext mutation contract - nothing writes config at pipeline entry now * Fixed three more comments the retirement had left stale, found by re-checking for the same claim * Marked ResolveSidecarBasePath's parquet fallback unreachable rather than deleting it mid-review See TODO-20260908_osprey_input_scores_retirement.md in pwiz-ai/todos Co-Authored-By: Claude --- .../Osprey/Osprey.Tasks/BlibOutputWriter.cs | 8 +++++--- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 11 ++++++----- .../Osprey/Osprey.Tasks/PipelineContext.cs | 12 ++++++++---- .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 18 ++++++++++-------- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs b/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs index e4c4edc336..9a978c503a 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/BlibOutputWriter.cs @@ -117,9 +117,11 @@ internal static void Write( // Pre-create source file IDs once. SpectrumSourceFiles.fileName carries // the ABSOLUTE path of each spectrum source file, matching BiblioSpec's // BlibBuild (BuildParser.insertSpectrumFilename resolves every name it - // is given to a full path). On a from-scores run the acquisition itself - // is not among the inputs, so the path is synthesized beside the parquet - // - the same rule the rescore hydrate uses. The golden and cross-impl + // is given to a full path). Every route names the acquisition on -i, so the + // path comes from the input itself - it does not have to EXIST for that, and + // on a join node it usually does not. A from-scores run used to arrive with + // the acquisition absent from the inputs entirely, and the path was + // synthesized beside the parquet. The golden and cross-impl // comparators key these strings by BASENAME, which is what keeps the // committed goldens machine-independent. SpectrumSourceFiles.idFileName // carries the library filename (Skyline expects this - Rust diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index b32b3ff95b..63989cb085 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -2264,11 +2264,12 @@ private static Action> BuildStage7PerRunSource( /// /// Build the file_name -> input_files index map used to pick the - /// right mzML path for the spectra-cache load + sibling - /// .calibration.json. For the worker, config.InputFiles was - /// synthesized from --input-scores parquet stems by Program.Main; - /// for in-process it's the user's -i mzML list. Either way the - /// stem matches the file_name keys in perFileEntries. + /// right data-file path for the spectra-cache load + sibling + /// .calibration.json. One source on every route: the user's -i list. + /// A worker's used to be SYNTHESIZED from --input-scores parquet stems by + /// Program.Main, which is the round trip that flag forced; both routes now + /// name the data files, so the stem matches the file_name keys in + /// perFileEntries without a second derivation. /// private static Dictionary BuildFileNameToIndex(IReadOnlyList inputFiles) { diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs b/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs index 54843e38a9..9870da762a 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PipelineContext.cs @@ -124,10 +124,14 @@ public sealed class PipelineContext /// must /// remain stable for the life of the run, so a worker can /// reproduce the same hash a straight-through invocation would - /// stamp into its parquet footers. Pipeline-populated fields - /// that do NOT feed those hashes (e.g. the worker-mode - /// population of InputFiles from --input-list) may be - /// written once at pipeline entry. Run-time state that is not parsed + /// stamp into its parquet footers. NOTHING is written to the config at + /// pipeline entry any more: it is complete when parsing ends + /// (OspreyCommandArgs.ToConfig, which is also where + /// --input-list is expanded into InputFiles). The carve-out + /// that stood here - pipeline-populated fields that do not feed the hashes + /// may be written once at entry - existed for the worker-mode synthesis of + /// InputFiles from --input-scores parquet stems, and it went + /// with that flag. Run-time state that is not parsed /// config (e.g. file parallelism) lives on /// instead. For per-file scratch that /// mutates hash-affecting fields (e.g. the MS2-calibrated diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index 1d8a558820..06cedd3af5 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -253,11 +253,14 @@ internal static SpectraWindowIndex EnsureSpectraCache(string inputFile, bool ser /// /// Resolve a path whose stem matches , used /// only as the base for sidecar file naming (the path itself need - /// not exist). In normal mode this is the input mzML; in - /// --task FirstPassFDR mode where InputFiles is empty we synthesize the - /// path from the matching .scores.parquet by replacing the - /// `.scores.parquet` suffix with `.mzML`. Mirrors the Rust - /// `synthetic_input_from_parquet` helper. + /// not exist). This is the input data file, on every route. + /// + /// The parquet-derived fallback below is UNREACHABLE now and is kept + /// only because removing it is a behaviour change that belongs in its own + /// commit. It existed for --task FirstPassFDR, which took + /// --input-scores and so arrived with InputFiles empty; every + /// task now requires --input, and the fileName keys are + /// derived from those same inputs, so the loop always matches. /// /// Lives here rather than on because /// needs the same resolution to find a @@ -284,9 +287,8 @@ internal static string ResolveSidecarBasePath( } } } - // --task FirstPassFDR fallback: derive a synthetic mzML path from the - // matching parquet stem so all the existing sidecar path - // helpers keep working without conditional branches. + // Unreachable since --input-scores retired - see the remarks. Left in + // place rather than deleted mid-review-round. if (perFileParquetPaths != null && perFileParquetPaths.TryGetValue(fileName, out string parquetPath)) { From c0da9e5af8fa17f707d90bc90d0ed0c0cf6c07d1 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 22:06:37 -0700 Subject: [PATCH 27/30] Retired the scores-parquet fallback and fixed the review blockers * Made a missing .scores-reconciled.parquet a hard failure on every arm, and deleted EffectiveScoresPathFromScoresPath so the parquet a task reads is decided by the task rather than by a disk probe * Refused --task FirstPassFDR admission to the Stage 7 streamed join, which a re-run over a completed directory silently satisfied * Refused duplicate input stems, gated the parquet stand-in on tasks that start after Stage 4, and made the reconciled-shape probe answer instead of throwing * Asserted in regression.ps1 that no consumer pulled the whole pool See TODO-20260908_osprey_stage7_straightthrough_stream.md in pwiz-ai/todos Co-Authored-By: Claude --- pwiz_tools/Osprey/Osprey-workflow.html | 6 +- .../Osprey/Osprey.IO/ParquetScoreCache.cs | 80 ++-- .../Osprey/Osprey.Tasks/Pass2FdrSidecar.cs | 26 +- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 428 ++++++++++-------- .../Osprey/Osprey.Tasks/PerFileScoringTask.cs | 6 +- .../Osprey/Osprey.Tasks/ScoringTaskShared.cs | 107 ++++- .../Osprey/Osprey.Tasks/SecondPassFdrTask.cs | 68 ++- pwiz_tools/Osprey/Osprey.Test/IOTest.cs | 69 ++- .../Osprey.Test/LibraryFragmentReleaseTest.cs | 4 +- .../Osprey.Test/PipelineMembershipTest.cs | 65 ++- pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs | 44 +- pwiz_tools/Osprey/Osprey/Program.cs | 76 +++- pwiz_tools/Osprey/README.md | 46 +- .../Osprey/docs/00-pipeline-architecture.md | 14 +- .../Osprey/docs/14-intermediate-files.md | 23 +- .../Osprey/docs/15-hpc-scoring-split.md | 10 +- pwiz_tools/Osprey/docs/16-determinism.md | 7 +- pwiz_tools/Osprey/regression.ps1 | 42 +- 18 files changed, 784 insertions(+), 337 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey-workflow.html b/pwiz_tools/Osprey/Osprey-workflow.html index 750abd1b3a..88613e60ff 100644 --- a/pwiz_tools/Osprey/Osprey-workflow.html +++ b/pwiz_tools/Osprey/Osprey-workflow.html @@ -395,7 +395,7 @@

Osprey DIA pipeline workflow

▸ --task FirstPassFDR join · 1 node · holds O(distinct), never O(runs x entries) - in  <stem>.scores.parquet, <stem>.calibration.json (every run, via --input-scores) + in  <stem>.scores.parquet, <stem>.calibration.json (every run, derived from --input) out <stem>.1st-pass.fdr_scores.bin, .1st-pass.stratum.json, <stem>.reconciliation.json · validity <out>.FirstPassFDR.osprey.task out <blib-stem>.1st-pass.fdr_experiment.bin, .1st-pass.retained_base_ids.bin (compaction key), <stem>.1st-pass.model.json relay: every experiment-wide file to EVERY downstream node @@ -462,7 +462,7 @@

Osprey DIA pipeline workflow

▸ --task PerFileRescoring per-run fan-out · reads its own runs + the experiment baseline only - in  <stem>.scores.parquet (via --input-scores) + .1st-pass.fdr_scores.bin, .reconciliation.json, .calibration.json, .spectra.bin + in  <stem>.scores.parquet (derived from --input) + .1st-pass.fdr_scores.bin, .reconciliation.json, .calibration.json, .spectra.bin in  <blib-stem>.1st-pass.fdr_experiment.bin, .1st-pass.retained_base_ids.bin, <stem>.1st-pass.model.json · resident baseline out <stem>.scores-reconciled.parquet, .2nd-pass.fdr_decoys.bin, .2nd-pass.fdr_scores.bin (pass-2 worker) · validity <out>.PerFileRescoring.osprey.task relay: the run's own set + every experiment-wide file @@ -506,7 +506,7 @@

Osprey DIA pipeline workflow

▸ --task SecondPassFDR join · 1 node · final aggregation - in  <stem>.scores-reconciled.parquet (via --input-scores; falls back to <stem>.scores.parquet), .2nd-pass.fdr_scores.bin (worker) + in  <stem>.scores-reconciled.parquet (every run - one parquet per run, no .scores.parquet fallback), .2nd-pass.fdr_scores.bin (worker) in  <blib-stem>.1st-pass.fdr_experiment.bin, <stem>.1st-pass.model.json, .1st-pass.stratum.json, <stem>.reconciliation.json, .calibration.json out <output>.blib, <blib-stem>.2nd-pass.fdr_experiment.bin, <stem>.2nd-pass.fdr_scores.bin (where no worker ran) · validity <out>.SecondPassFDR.osprey.task holds O(distinct) · folds the runs one at a time, rebuilding each from its own artifacts and dropping it diff --git a/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs b/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs index c9ed15cf77..0107ec7c11 100644 --- a/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs +++ b/pwiz_tools/Osprey/Osprey.IO/ParquetScoreCache.cs @@ -246,13 +246,7 @@ private static ParquetSchema BuildWriteSchema(DataField[] featureFields, /// public static bool IsSubsetWithoutScoreIndex(string path) { - if (!File.Exists(path)) - return false; - var footer = LoadFooterMetadata(path); - footer.TryGetValue(@"osprey.reconciled", out string marker); - if (!string.Equals(marker, RECONCILED_SURVIVORS, StringComparison.Ordinal)) - return false; - return !HasColumn(path, FIELD_SCORE_INDEX.Name); + return ProbeReconciledSurvivorShape(path, out bool hasScoreIndex) && !hasScoreIndex; } /// @@ -275,13 +269,59 @@ public static bool IsSubsetWithoutScoreIndex(string path) /// public static bool IsCurrentReconciledSurvivorSubset(string path) { + return ProbeReconciledSurvivorShape(path, out bool hasScoreIndex) && hasScoreIndex; + } + + /// + /// The one open behind and + /// : does carry the + /// current marker, and does it have the + /// score_index column? Returns false for anything this build cannot read as a + /// reconciled survivor parquet, including a file that is absent, empty, half-written + /// or foreign. + /// + /// Answers, never throws. Both callers are boolean predicates whose + /// documented false covers "not readable in that shape", and five call sites branch on + /// them - so ONE zero-length or partially-written parquet turning a predicate into an + /// unhandled stack trace pre-empts SecondPassFdrTask's named, file-listing + /// refusal, which is the message the operator is supposed to get. + /// already wrapped the identical call, so the + /// convention existed before this did. + /// + /// One open, not two. The footer and the schema come off the same reader. + /// Read separately they were two opens per file per call, uncached across five call + /// sites - order 4,460 parquet opens on a 446-run cohort before any work begins, and + /// typically on a network artifact directory. + /// + private static bool ProbeReconciledSurvivorShape(string path, out bool hasScoreIndex) + { + hasScoreIndex = false; if (string.IsNullOrEmpty(path) || !File.Exists(path)) return false; - var footer = LoadFooterMetadata(path); - footer.TryGetValue(@"osprey.reconciled", out string marker); - if (!string.Equals(marker, RECONCILED_SURVIVORS, StringComparison.Ordinal)) + try + { + using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var reader = RunSync(ParquetReader.CreateAsync(stream))) + { + reader.CustomMetadata.TryGetValue(@"osprey.reconciled", out string marker); + if (!string.Equals(marker, RECONCILED_SURVIVORS, StringComparison.Ordinal)) + return false; + foreach (var f in reader.Schema.GetDataFields()) + { + if (!string.Equals(f.Name, FIELD_SCORE_INDEX.Name, StringComparison.Ordinal)) + continue; + hasScoreIndex = true; + break; + } + return true; + } + } + catch (Exception ex) when (!(ex is OutOfMemoryException)) + { + // Unreadable IS "not a current reconciled survivor parquet". The caller that + // cares which file it was names it; see SecondPassFdrTask.UnusableReconciledParquets. return false; - return HasColumn(path, FIELD_SCORE_INDEX.Name); + } } /// Whether a parquet's schema carries a column by this name. @@ -1874,24 +1914,6 @@ public static string ReconciledPathFromScoresPath(string scoresPath) return scoresPath; } - /// - /// The path a post-Stage-6 reader (Stage 7 feature reload, resume / - /// --task SecondPassFDR) should consume for a given original - /// .scores.parquet path: the reconciled sibling when it exists - /// on disk, otherwise the original. This per-file selection is the - /// read-side contract that makes the separate-reconciled-file design - /// byte-equivalent to the former in-place overwrite: files that had - /// reconciliation work read the reconciled bytes (which used to be - /// written over the original), while files with no Stage 6 work -- which - /// PerFileRescoreTask deliberately skips, leaving no reconciled - /// file -- read the untouched original (which used to be left in place). - /// - public static string EffectiveScoresPathFromScoresPath(string scoresPath) - { - string reconciled = ReconciledPathFromScoresPath(scoresPath); - return File.Exists(reconciled) ? reconciled : scoresPath; - } - /// /// Check if an existing Parquet file's custom metadata matches the expected values. /// Returns true if all expected keys exist with matching values. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs index cf16d9b15f..2c27c279d4 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/Pass2FdrSidecar.cs @@ -1486,9 +1486,13 @@ internal static void WritePass2ExperimentSidecar( var byEntryId = new Dictionary(); try { - string parquetPath = - ParquetScoreCache.EffectiveScoresPathFromScoresPath( - perFileParquetPaths[fileName]); + // The RECONCILED parquet, derived rather than probed. This runs both + // in-process - where the published map holds Stage 4 paths, because + // Stage 1-4 ran here - and on a --task SecondPassFDR node, where it + // already holds reconciled ones; the derivation is idempotent, so one + // expression states the same intent on both routes. + string parquetPath = ParquetScoreCache.ReconciledPathFromScoresPath( + perFileParquetPaths[fileName]); ParquetScoreCache.ReadFdrStubScalars(parquetPath, (entryId, charge, isDecoy, coelutionSum, modseq) => { @@ -2026,7 +2030,7 @@ void BeginFile(string fileKey) // The parquet lookup is established by the validation loop above (every file // has a parquet path or this method already returned false), and resolved // HERE so a key miss cannot be reported as a parquet failure by the reader. - string effectiveParquetPath = ParquetScoreCache.EffectiveScoresPathFromScoresPath( + string effectiveParquetPath = ParquetScoreCache.ReconciledPathFromScoresPath( perFileParquetPaths[fileKey]); // currentKey/currentEntries are staged by BeginFile now, on every path. // Read from the path the validation loop above checked with IsCurrentFormat, @@ -2385,13 +2389,12 @@ private static void ComputePass2Resident( kvp.Key, kvp.Value.Count)); continue; } - // Read the RECONCILED parquet (Stage 6's rescored - // features) when it exists; fall back to the original - // Stage 4 parquet for files that had no reconciliation - // work (no reconciled sibling was written). The - // perFileParquetPaths map holds original paths. + // Read the RECONCILED parquet - Stage 6's rescored features - which it + // writes for every run, so this is a derivation and not a preference. + // The published map holds Stage 4 paths in-process and reconciled ones on + // a --task SecondPassFDR node; the derivation is idempotent over both. string effectiveParquetPath = - ParquetScoreCache.EffectiveScoresPathFromScoresPath(parquetPath); + ParquetScoreCache.ReconciledPathFromScoresPath(parquetPath); Dictionary featByScoreIndex; try { @@ -3582,8 +3585,7 @@ private bool WriteCore(string fileName, Action write) { TaskValiditySidecar.Write(pass2Path, _taskName, OspreyVersion.Current, _taskValidityKey, - new[] { ParquetScoreCache.EffectiveScoresPathFromScoresPath( - ParquetScoreCache.GetScoresPath(inputFile)) }); + new[] { ParquetScoreCache.GetReconciledScoresPath(inputFile) }); } catch (Exception ex) when (!(ex is OutOfMemoryException)) { diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index 63989cb085..c8ab27780e 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -166,11 +166,15 @@ public override bool IsIncluded(PipelineContext ctx) // their data files now, so the task says which worker this is, which is the only // thing that ever actually decided it. // - // StopAfterStage5 means that boundary whatever the inputs look like: it used to - // be checked on one route only, which was enough while --task FirstPassFDR was - // its only setter, and --task ModelDiagnostics sets it too - so this task ran - // anyway, demanded CompactedEntries that a diagnostics fold never publishes, and - // failed the run AFTER the report it was asked for had been written. + // StopAfterStage5 means that boundary whatever the inputs look like, and it is + // checked on every route rather than one - it used to be checked on one only, + // which failed a run AFTER writing the report it was asked for. + // + // --task FirstPassFDR is its ONLY setter (Program.cs has the single assignment). + // The paragraph here said ModelDiagnostics sets it too; it does not, and the same + // claim had been copied into docs/15-hpc-scoring-split.md's truth table and a unit + // test helper that built the config to match. ModelDiagnostics sets none of the + // three flags, so it is a member of every task and suppresses artifact WRITES. return c.SelectedTask == HpcTask.PerFileRescore || (!c.NoJoin && !c.StopAfterStage5 && !c.ExpectReconciledInput); } @@ -685,19 +689,22 @@ public override bool Rehydrate(PipelineContext ctx) // buffer stays at 1st-pass RTs and SecondPassFDR (which reads ApexRt/ // StartRt/EndRt/BoundsArea straight off these entries) writes 1st-pass // RTs into the final blib instead of the Stage 6 reconciled values. - // Files with no reconciled sibling on disk are no-work files; a fresh - // run leaves their entries at 1st-pass too, so they are left unchanged. + // A run with no current reconciled parquet is a FAILURE here, not a no-work + // file. Stage 6 writes one for every run - WriteUnchangedReconciled covers the + // run it did no work on - so absence means the write never landed (P13: a + // phase with nothing to say writes the file anyway, precisely so absence stays + // unambiguous). The Stage 4 parquet is not a substitute: it holds pre- + // reconciliation boundaries and none of the gap-fill rows. // - // Refill first when Stage 5 released the survivors (issue #4526). A resume - // that re-runs Stage 5 lands here with an EMPTY buffer, and overlaying onto - // empty lists produces an almost-empty blib instead of failing. // Eager here, unlike Run: Rehydrate itself only ever executes because a // consumer pulled this milestone, so the work is already lazy. - var resumeLoader = StreamedSurvivorLoader(ctx); - if (resumeLoader != null) - MaterializeAllSurvivors(_perFileEntries, resumeLoader, ctx); - OverlayReconciledIntoFiles(_perFileEntries, CurrentReconciledPaths(ctx), - ctx.Get().Value); + MaterializeAllResumedFiles(_perFileEntries, StreamedSurvivorLoader(ctx), + CurrentReconciledPaths(ctx), + // Lazy for the reason BuildResumePerRunSource states: with a survivor + // loader every run takes the load branch and the gap-fill map - order + // 10 GB of envelope JSON at 446 runs - is never read. + new Lazy>>( + () => ctx.Get().Value), ctx); ctx.Publish(new RescoredEntries(_perFileEntries)); return true; @@ -1095,8 +1102,10 @@ void WriteAnswer(string fileName, IReadOnlyList records, // PerFileRescoring. Stamping SecondPassFDR's key here would leave a file that // outlives the inputs it was computed from, which is the one thing a resume // cannot detect by looking. - var stampInputs = new[] { ParquetScoreCache.EffectiveScoresPathFromScoresPath( - ParquetScoreCache.GetScoresPath(inputFile)) }; + // The RECONCILED parquet, named: these artifacts were computed from the file + // this same task wrote moments ago, so that is what they must be stamped + // against. Its Stage 4 sibling is a different population. + var stampInputs = new[] { ParquetScoreCache.GetReconciledScoresPath(inputFile) }; TaskValiditySidecar.Write(decoysPath, taskName, OspreyVersion.Current, taskValidityKey, stampInputs); TaskValiditySidecar.Write(pass2Path, taskName, OspreyVersion.Current, @@ -1530,7 +1539,10 @@ private sealed class RescorePassInputs inputs.Pass2Worker.CompeteStampAndWrite( fileName, FdrScoresSidecar.Pass1Path(inputFile), - ParquetScoreCache.EffectiveScoresPathFromScoresPath(parquetPath), + // The reconciled parquet the refusal above is worded for. Derived rather + // than probed: this worker wrote it earlier in this same task, so its + // absence is a failure to report, not a reason to read Stage 4's. + ParquetScoreCache.ReconciledPathFromScoresPath(parquetPath), fdrEntries); } @@ -2105,19 +2117,18 @@ private static void MaterializeAllFromSource( /// /// The per-run source Stage 7 folds through on the STRAIGHT-THROUGH resume: bring ONE - /// run's list to the post-rescore state the two whole-run loops in + /// run's list to the post-rescore state the whole-run loop in /// would have brought every run to, so the join holds one run at a time. /// - /// Its body is those loops' own per-file halves, called in the same order rather - /// than reimplemented: refill the released survivors, then overlay this run's reconciled - /// parquet. Nothing in either half reads another run's entries, so run-at-a-time is the - /// same work in the same order as all-runs-then-all-runs - which is why the streamed and - /// resident arms write byte-identical output, and why the fix is a call-shape change - /// rather than a second implementation. + /// Its body is - that loop's own per-file + /// half, called rather than reimplemented. Nothing in it reads another run's entries, + /// so run-at-a-time is the same work in the same order as all-runs - which is why the + /// streamed and resident arms write byte-identical output, and why the fix is a + /// call-shape change rather than a second implementation. /// /// RE-ENUMERABLE, which is the property - /// needs: a fold pass drops each run's list, so a second pass finds it empty, and both - /// halves here rebuild an empty list from disk identically. Unconditionally so, unlike + /// needs: a fold pass drops each run's list, so a second pass finds it empty, and the + /// half here rebuilds an empty list from disk identically. Unconditionally so, unlike /// : a resume runs no rescore, so no run's list ever /// holds state that is not already on disk, and there is nothing a drop can lose. /// @@ -2145,23 +2156,27 @@ private Action> BuildResumePerRunSource(PipelineContext c // the driver stamps a fresh validity sidecar onto every declared output that merely // exists. The answer travels; the question does not. var reconciledPaths = CurrentReconciledPaths(ctx); - var gapFill = ctx.Get().Value; + // LAZY, and on this arm usually never read at all. PerFileGapFillForRescore is + // published DEFERRED, and its own header records what pulling it costs: "dotTrace + // 69.7s total in ReadGapFillAndCalibrations ... an in-code probe at 92.7s; and a + // night run's log gap at 111s" - order 10 GB of envelope JSON at 446 runs. Pulling + // it here made the whole-run read unconditional inside a method whose surrounding + // comment says the work should wait for the consumer that folds. It is needed ONLY + // by the overlay branch, which a run with a survivor loader does not take. + var gapFill = new Lazy>>( + () => ctx.Get().Value); // Says which shape Stage 7 took, for the reason its --task SecondPassFDR sibling // gives: without it the only evidence is a memory profile, and "the gate is green so // the new path must have run" is the inference that lets a resident path pass as a // streamed one. ctx.LogInfo(string.Format( - @"Second-pass join: folding over {0} run(s), each rebuilt from its own first-pass " + - @"survivors and dropped (no all-runs survivor pool). {1} of {0} run(s) carry a " + - @"current reconciled parquet to overlay; the rest keep their 1st-pass boundaries, " + - @"as they would on a fresh run.", + @"Second-pass join: folding over {0} run(s), each rebuilt from its own " + + @"reconciled parquet and dropped (no all-runs survivor pool). {1} of {0} run(s) " + + @"carry a current reconciled parquet; a run without one is an error, not a " + + @"run that keeps its 1st-pass boundaries.", _perFileEntries.Count, reconciledPaths.Count)); return (fileName, survivors) => - { - MaterializeFileSurvivors(fileName, survivors, loader, ctx); - OverlayReconciledIntoFile(fileName, survivors, reconciledPaths, gapFill, - canonicalize: true); - }; + MaterializeResumedFile(fileName, survivors, loader, reconciledPaths, gapFill, ctx); } private static Action> BuildStage7PerRunSource( @@ -2226,18 +2241,26 @@ private static Action> BuildStage7PerRunSource( @"Second-pass join hydrate: no scores parquet path published for {0}", fileName)); } - // Resolve to the RECONCILED sibling, exactly as every other reader on this leg - // does. perFileParquetPaths holds whatever --input-scores named, and the - // documented fallback form names .scores.parquet - so rebuilding from the - // path verbatim would give this arm the PRE-reconciliation rows (first-pass - // boundaries, no Stage-6 gap-fill) while the resident arm on the same command - // line reads the reconciled file. Two different .blib files from one command - // line is precisely what the byte-identity oracle exists to prevent, and it - // would not have caught it: both arms would be self-consistent. - string effectivePath = - ParquetScoreCache.EffectiveScoresPathFromScoresPath(parquetPath); + // The published path IS the reconciled parquet on this leg: it is reached only + // under --task SecondPassFDR, and ScoringTaskShared.ScoresPathsForInputs + // resolves that task's paths to the reconciled artifact by task membership + // rather than by probing disk. Asserted rather than re-resolved, because + // rebuilding from a Stage 4 path would give this arm the PRE-reconciliation + // rows (first-pass boundaries, no Stage-6 gap-fill) while the resident arm on + // the same command line reads the reconciled file. Two different .blib files + // from one command line is precisely what the byte-identity oracle exists to + // prevent, and it would not have caught it: both arms would be self-consistent. + if (!ParquetScoreCache.IsReconciledScoresPath(parquetPath)) + { + throw new InvalidDataException(string.Format( + @"Second-pass join hydrate: run '{0}' published '{1}', which is not a " + + @".scores-reconciled.parquet. The join rebuilds every run from its " + + @"reconciled artifact; a Stage 4 path here would silently produce " + + @"1st-pass boundaries with no gap-fill rows.", + fileName, parquetPath)); + } bool overlayFirstPass = !haveSecondPass.Contains(fileName); - RescoreHydration.RefillOneRunSurvivors(fileName, effectivePath, survivors, + RescoreHydration.RefillOneRunSurvivors(fileName, parquetPath, survivors, retainedBaseIds, overlayFirstPass ? experimentRecords.Value : null, (name, path) => ParquetScoreCache.LoadFdrStubsFromParquet(path, null, sequencePool), overlayFirstPass); @@ -2664,8 +2687,10 @@ private sealed class RescoredPoolPlan /// plan is a refill and nothing else). EMPTY is not null: a rescore that ran and /// skipped every file still overlays, exactly as a cold run does. /// file name -> the .scores-reconciled.parquet - /// judged CURRENT while Run still held the answer. Files absent from this map keep - /// their 1st-pass boundaries. + /// judged CURRENT while Run still held the answer. A file absent from this map is a + /// run Stage 6 did not persist, and it is a hard failure - see + /// . Null only on the refill-only plan, which + /// does not overlay at all. /// The planner's per-file gap-fill targets, for the /// overlay. /// Per file, the entry_ids whose scores the rescore @@ -2776,31 +2801,16 @@ private Action> BuildRunPerRunSource( return (fileName, entries) => { var plan = PoolPlanForBuild(); - // THE ONE RUN THE FOLD CANNOT SERVE, and it fails rather than folding it. + // A run with no current reconciled parquet fails in MaterializeRescoredFile + // below, on BOTH arms, rather than being caught here for the fold alone. // ExecuteRescore drops a run's entries only when its reconciled parquet reached - // disk, KEEPING them when the write no-opped or failed - because in that case - // those entries are the only copy of the rescore. A fold drops every run it - // hands over, so streaming such a run would discard that copy and the next pass - // would rebuild it from the Stage 4 parquet: a blib silently carrying 1st-pass - // boundaries for one run, from a run that exits 0. The resident build survives - // it by never dropping anything, which is why this is new here rather than a - // defect being uncovered. + // disk, KEEPING them when the write no-opped or failed - so on the resident arm + // that run's rescore survives in memory and the output looks right while + // nothing was persisted. A fold drops every run it hands over, so the same run + // would be rebuilt from a parquet that is not there. Neither is a run that + // finished: one arm cannot be allowed to treat as routine what the other has + // to fail on, or the missing artifact is a property of the arm. // - // RescoredFiles null means no rescore ran at all (the self-gated refill-only - // plan), and then no run has - or needs - a reconciled parquet: every one is - // rebuilt from its Stage 4 parquet plus its 1st-pass sidecar, repeatably. It is - // only a run that WAS rescored and has no current reconciled parquet that has - // state nothing on disk holds. - if (plan.RescoredFiles != null && - (plan.ReconciledPaths == null || !plan.ReconciledPaths.ContainsKey(fileName))) - { - throw new InvalidDataException(string.Format( - @"Second-pass join: run '{0}' has no current .scores-reconciled.parquet, " + - @"so its re-scored survivors exist only in memory and the per-run fold " + - @"cannot rebuild them. Stage 6 did not persist this run - it logged a " + - @"warning when the write no-opped or failed. Re-run Stage 6 for it.", - fileName)); - } // CLEARED here rather than relying on the caller having dropped the run. // StreamFiles does drop it, but MaterializeFile leaves that to its caller, and // the whole repeatability argument above rests on the list being empty - so the @@ -2872,20 +2882,27 @@ private void BuildRescoredPool(PipelineContext ctx) private void MaterializeRescoredFile(PipelineContext ctx, RescoredPoolPlan plan, string fileName, List entries) { - // ONE parquet, not two. When this file's reconciled parquet was judged - // current, it already holds the survivor subset with Stage 6's boundaries - // applied and the gap-fill rows merged - so reading it makes both the - // Stage 4 read and the overlay that put those values back unnecessary - // (#4486). Stage 6 originally OVERWROTE the Stage 4 parquet, which is why - // one read used to give both; splitting the files left Stage 7 reading one - // for the rows and the other for the values. - string reconciledPath = null; - plan.ReconciledPaths?.TryGetValue(fileName, out reconciledPath); - bool loadedReconciled = reconciledPath != null && entries.Count == 0; - MaterializeFileSurvivors(fileName, entries, plan.Loader, ctx, - loadedReconciled ? reconciledPath : null); + // The refill-only plan FIRST, because it is the one route here that legitimately + // never reads a reconciled parquet - not a run missing one. The resident arm does + // nothing at all on this route: it leaves the buffer as Stage 5 compacted it and + // SecondPassFDR reloads the rescored features from the reconciled parquets by + // identity, so overlaying here would apply Stage-6 boundaries twice (see the + // RefillOnly call site). Its one job is to put back what FirstPassFDR released. if (plan.RescoredFiles == null) + { + MaterializeFileSurvivors(fileName, entries, plan.Loader, ctx); return; + } + // ONE parquet, not two. This file's reconciled parquet already holds the survivor + // subset with Stage 6's boundaries applied and the gap-fill rows merged - so + // reading it makes both the Stage 4 read and the overlay that put those values + // back unnecessary (#4486). Stage 6 originally OVERWROTE the Stage 4 parquet, + // which is why one read used to give both; splitting the files left Stage 7 + // reading one for the rows and the other for the values. + string reconciledPath = ReconciledPathOrFail(fileName, plan.ReconciledPaths, ctx); + bool loadedReconciled = entries.Count == 0; + MaterializeFileSurvivors(fileName, entries, plan.Loader, ctx, + loadedReconciled ? reconciledPath : null); // BEFORE the overlay, which appends gap-fill rows: the planner's indices // address the survivor list as loaded, and appending shifts nothing but // would be indexed if the reset ran after. The overlay preserves Score / @@ -2896,7 +2913,7 @@ private void MaterializeRescoredFile(PipelineContext ctx, RescoredPoolPlan plan, // copy of the gap-fill rows already merged into them. if (!loadedReconciled) { - OverlayReconciledIntoFile(fileName, entries, plan.ReconciledPaths, + OverlayReconciledIntoFile(fileName, entries, reconciledPath, plan.GapFill?.Value, canonicalize: false); } } @@ -2925,10 +2942,15 @@ private IReadOnlyDictionary CurrentReconciledPaths(PipelineConte } /// - /// Refill every file whose survivor list was released, leaving files that already - /// hold entries untouched so a second call is a no-op. Throws if any file's parquet - /// or 1st-pass sidecar cannot be read - Stage 5 wrote both, so a failure here is a - /// fault rather than an absence. + /// Bring every file to its post-rescore state on a RESUME: the whole-run shape of + /// , which is also what + /// folds through one run at a time. One loop + /// where there were two - a refill pass over every file followed by an overlay pass + /// over every file - because both passes now resolve to the same single parquet, so + /// splitting them only decided how long each file's list stayed alive. + /// + /// Throws if any file's artifacts cannot be read - Stage 5 and Stage 6 wrote + /// them, so a failure here is a fault rather than an absence. /// /// Logs the fault and sets the exit code before throwing. The throw is what the /// deferred needs (a pull has no bool channel back to @@ -2938,15 +2960,22 @@ private IReadOnlyDictionary CurrentReconciledPaths(PipelineConte /// the message and a stack trace. Returning false and letting Stage 7 build a partial /// pool is the one option not on the table. /// - private static void MaterializeAllSurvivors( + private static void MaterializeAllResumedFiles( List>> perFileEntries, - FirstPassSurvivorLoader loader, PipelineContext ctx) + FirstPassSurvivorLoader loader, + IReadOnlyDictionary reconciledPaths, + Lazy>> gapFill, PipelineContext ctx) { // Reported, not silent: this is a per-file parquet + sidecar read across every file // in the run, landing in the quiet window where Stage 7 starts (or, on resume, in // the rehydrate) with nothing else printing. An unreported sequential loop of // exactly this shape has twice read as a hung run in this codebase (#4513, // Pass2FdrSidecar). Console-only. + // The SAME label the cold arm's loop uses, because it is now the same operation on + // the same artifact - one name, so a log reads the same whichever arm produced it, + // and the resident-join landmark in ai/scripts/Osprey/CHS/README.md keeps meaning + // what it says. It names WHAT is rebuilt (the first-pass survivor subset), not + // which file supplied the rows. using (var progress = new ProgressReporter(string.Format( @"Rebuilding first-pass survivors from {0} file(s)", perFileEntries.Count), perFileEntries.Count)) @@ -2955,16 +2984,102 @@ private static void MaterializeAllSurvivors( foreach (var kv in perFileEntries) { progress.Report(++done); - MaterializeFileSurvivors(kv.Key, kv.Value, loader, ctx); + MaterializeResumedFile(kv.Key, kv.Value, loader, reconciledPaths, gapFill, ctx); } } } + /// + /// Bring ONE file's list to its post-rescore state on a RESUME, from that file's OWN + /// .scores-reconciled.parquet. The per-file half both resume arms call - the + /// whole-run loop above and the per-run source Stage 7 folds through - so the two + /// cannot disagree about what a run's post-rescore state is. + /// + /// ONE parquet, not two, which is the Boundary 3 -> 4 contract: the reconciled + /// parquet holds the survivor subset with Stage 6's boundaries already applied and the + /// gap-fill rows already merged in canonical position, so reading it makes both the + /// Stage 4 read and the overlay that put those values back unnecessary. It also lands + /// the rows in canonical order ( sorts, and its + /// callers must not re-order), where the overlay had to sort afterwards because it + /// APPENDED the gap-fill rows. + /// + /// The overlay survives for one case, and it is not a fallback for a missing + /// artifact: a list that still HOLDS its entries has its rows already, and re-reading + /// them would be the duplicate build OSPREY_STAGE6_STREAM_SURVIVORS=0 exists to + /// avoid - so those rows take the reconciled values through the overlay instead. Both + /// routes read the same one parquet. + /// + private static void MaterializeResumedFile(string fileName, List entries, + FirstPassSurvivorLoader loader, + IReadOnlyDictionary reconciledPaths, + Lazy>> gapFill, PipelineContext ctx) + { + string reconciledPath = ReconciledPathOrFail(fileName, reconciledPaths, ctx); + if (loader != null) + { + // CLEARED here rather than relying on the caller having dropped the run - the + // same establish-your-own-precondition BuildRunPerRunSource states, and for the + // same reason. StreamFiles does drop it, but RescoredEntries.MaterializeFile + // invokes the source WITHOUT clearing and leaves dropping to its caller, whose + // own doc names "merely finished one of several passes over it" as legitimate. + // On a non-empty list the overlay below would APPEND this run's gap-fill rows a + // second time - duplicating precursors in the pool Stage 7 writes the .blib + // from, silently, exit 0. That is the duplication BuildRunPerRunSource's comment + // records once exiting a straight-through Stellar run on AssertSidecarDescribesPool. + // + // Guarded on the loader, which is what makes the clear safe: a run cleared here + // is rebuilt from disk on the next line. Where there is NO loader the entries + // are the only copy - the OSPREY_STAGE6_STREAM_SURVIVORS=0 resident oracle - + // and they take the reconciled values through the overlay instead. + entries.Clear(); + MaterializeFileSurvivors(fileName, entries, loader, ctx, reconciledPath); + return; + } + OverlayReconciledIntoFile(fileName, entries, reconciledPath, gapFill.Value, + canonicalize: true); + } + + /// + /// This run's CURRENT .scores-reconciled.parquet, or a hard failure naming the + /// run. Absence is a fault, never a case to work around: Stage 6 writes this artifact + /// for every run - covers the run it did no work + /// on - so a run missing from the current set means the write never landed or what + /// landed is not what this run would produce. That is principle P13 read from the + /// consumer's end: the producer writes unconditionally so that absence stays + /// unambiguous, and the consumer is then entitled to treat it as failure. + /// + /// The Stage 4 parquet is NOT a substitute, which is what this method exists to + /// stop being re-derived. It carries pre-reconciliation boundaries and none of the + /// gap-fill rows, so substituting it writes 1st-pass boundaries for that run into the + /// blib from a process that exits 0 - and on the resident arm, where the entries are + /// still in memory, it hides that the run was never persisted at all. + /// + private static string ReconciledPathOrFail(string fileName, + IReadOnlyDictionary reconciledPaths, PipelineContext ctx) + { + string reconciledPath = null; + if (reconciledPaths != null && reconciledPaths.TryGetValue(fileName, out reconciledPath)) + return reconciledPath; + // Logged as well as thrown, for the reason MaterializeAllResumedFiles gives: the + // top-level handler prints the message and a stack trace, where an operator needs + // the run named in the log beside the phase that failed. + string error = string.Format( + @"Second-pass join: run '{0}' has no current .scores-reconciled.parquet. Stage 6 " + + @"writes one for every run, so this run was not persisted - the write no-opped, " + + @"failed, or produced a file this run rejects as stale. Its .scores.parquet is " + + @"not a substitute: it holds 1st-pass boundaries and none of the gap-fill rows. " + + @"Re-run Stage 6 for it.", + fileName); + ctx.LogError(error); + ctx.ExitCode = 1; + throw new InvalidDataException(error); + } + /// /// Refill ONE file's survivor list, or leave it alone when it already holds entries - /// so a second call is a no-op. The per-file half of - /// , separate because the resume overlay loop - /// needs it per file with a reconciled-parquet override. + /// so a second call is a no-op. Every caller now supplies the reconciled-parquet + /// override except the refill-only plan, which deliberately does not overlay at all + /// (see ). /// private static void MaterializeFileSurvivors(string fileName, List entries, FirstPassSurvivorLoader loader, PipelineContext ctx, @@ -3053,100 +3168,57 @@ private static void ResetRescoredTargetsForFile(RescoredPoolPlan plan, string fi } /// - /// Bring EVERY file's list to its post-rescore state by overlaying that file's - /// .scores-reconciled.parquet, canonicalizing the order, and releasing the - /// re-fattened payload. This is the state a fresh - /// leaves behind, rebuilt from disk. + /// Bring ONE file's list to its post-rescore state by overlaying that file's + /// .scores-reconciled.parquet onto rows it ALREADY holds, canonicalizing the + /// order, and releasing the re-fattened payload. /// - /// Two callers, one body. The resume uses it because - /// the driver skipped when the reconciled parquets were already - /// valid. The streamed rescore uses it because it deliberately dropped each file's - /// entries after writing that file's parquet, so the - /// milestone SecondPassFDR reads has to be rebuilt at - /// the end (issue #4526). Sharing the body is what makes the streamed buffer - /// identical to the resumed one. + /// The route for a list that still has its entries. Where the list is empty the + /// callers load from that same reconciled parquet instead, which is one read rather + /// than a Stage 4 read plus this overlay - see + /// and . Either way exactly one parquet is read + /// per run, and it is the reconciled one. /// - /// The two callers differ in WHEN the reconciled parquets were judged current, - /// which is why that judgement is a parameter: the resume path asks now, the deferred - /// streamed build asks during and carries the answer (see - /// ). + /// The path is a parameter, already resolved and already judged CURRENT by + /// , because the two callers differ in WHEN that + /// judgement was made: the resume path asks now, the deferred streamed build asks + /// during and carries the answer (see + /// ). Validity, not mere existence - testing + /// File.Exists would accept a reconciled parquet this run REJECTED and + /// re-scored, overlaying another run's boundaries into this one's blib. /// - /// The shared per-file buffer to bring to its - /// post-rescore state, updated in place. - /// file name -> the reconciled parquet to overlay. - /// Files absent from the map keep their 1st-pass boundaries, matching a fresh run's - /// no-work files. + /// The run whose list is being brought forward. + /// That run's list, updated in place. + /// The run's current reconciled parquet. /// The planner's per-file gap-fill targets. - /// Re-sort each file by the canonical + /// Re-sort the file by the canonical /// (EntryId, Charge, ScanNumber, ParquetIndex) key. TRUE on resume, where the /// buffer has to be brought to the order a cold run ends in. FALSE on the streamed /// rebuild, which is REPRODUCING a cold run: a cold rescore appends gap-fill at the /// END of the list and never re-sorts, so sorting here would move those rows into /// EntryId order and change the buffer order Stage 7 writes its 2nd-pass sidecars /// in - which changes the protein-compact competition and the reported set. - private static void OverlayReconciledIntoFiles( - List>> perFileEntries, - IReadOnlyDictionary reconciledPaths, - IReadOnlyDictionary> gapFill, - bool canonicalize = true) - { - // Reported for the same reason as the survivor rebuild above: a per-file parquet - // read across the whole run, in the silent window after the parallel rescore. - using (var progress = new ProgressReporter(string.Format( - @"Overlaying reconciled results from {0} file(s)", perFileEntries.Count), - perFileEntries.Count)) - { - int done = 0; - foreach (var kv in perFileEntries) - { - progress.Report(++done); - OverlayReconciledIntoFile(kv.Key, kv.Value, reconciledPaths, gapFill, - canonicalize); - } - } - } - - /// - /// The per-file half of - overlay one file's - /// reconciled parquet, optionally canonicalize, and release the re-fattened payload. - /// Separate because the resume overlay loop also applies it per file, and the - /// whole-run loop is only one of its callers. - /// private static void OverlayReconciledIntoFile(string fileName, List entries, - IReadOnlyDictionary reconciledPaths, + string reconciledPath, IReadOnlyDictionary> gapFill, bool canonicalize) { - // Overlay this file's reconciled boundaries when the caller judged its - // .scores-reconciled.parquet present AND CURRENT; no-work files (none on - // disk) keep their 1st-pass boundaries, matching a fresh run. - // - // Validity, not mere existence. The rescore's own per-file gate - // (TryResumeRescoredFile) asks PerFileResumeDriver.IsCurrent, so testing - // File.Exists accepted a reconciled parquet this run would have REJECTED - // and re-scored - one left by a run with different reconciliation - // parameters, say. That overlays stale boundaries onto a cold run's buffer, - // which is worse than the no-work fallback of leaving 1st-pass values. - if (reconciledPaths != null && - reconciledPaths.TryGetValue(fileName, out string reconciledPath)) - { - IReadOnlyList gapFillForFile = null; - if (gapFill != null && gapFill.TryGetValue(fileName, out var gfList)) - gapFillForFile = gfList; - OverlayReconciledIntoBuffer(entries, reconciledPath, gapFillForFile); - } - // Canonical sort for EVERY file (incl. no-work files) so the WARM - // buffer order matches the order COLD establishes in - // RunPercolatorFdr, independent of whether the file was rescored. + IReadOnlyList gapFillForFile = null; + if (gapFill != null && gapFill.TryGetValue(fileName, out var gfList)) + gapFillForFile = gfList; + OverlayReconciledIntoBuffer(entries, reconciledPath, gapFillForFile); + // Canonical sort so the WARM buffer order matches the order COLD establishes + // in RunPercolatorFdr. Needed on THIS route because the overlay APPENDS the + // gap-fill rows; the load-from-reconciled route gets canonical order from the + // loader itself and does not re-sort. if (canonicalize) SortFileEntriesCanonical(fileName, entries); // Same release the rescore path does once a file's reconciled parquet is // on disk: the overlay above re-fattened this file's entries straight // from that parquet, and holding those arrays for every file is the - // O(files) Stage-6 growth term. A no-work file was never fattened, so - // this is a no-op there. Nothing downstream reads them off the buffer - - // SecondPassFDR's 2nd pass reloads PIN features from the reconciled parquet - // by identity - and this leaves the same buffer shape COLD leaves. + // O(files) Stage-6 growth term. Nothing downstream reads them off the + // buffer - SecondPassFDR's 2nd pass reloads PIN features from the + // reconciled parquet by identity - and this leaves the same buffer shape + // COLD leaves. ReleaseRescoredPayload(entries); } @@ -3155,12 +3227,14 @@ private static void OverlayReconciledIntoFile(string fileName, List en /// the exact order a COLD run establishes via /// FirstPassFdrTask.RunPercolatorFdr (run by SecondPassFDR's 2nd-pass, /// which a WARM straight-through resume skips when the .2nd-pass sidecars - /// are already valid on disk). Both resume paths apply this to EVERY file's - /// list, including no-work files with no reconciled parquet, so the WARM buffer - /// order matches COLD regardless of whether the file was rescored -- otherwise - /// SecondPassFDR's BuildSharedBoundaries could iterate a different order and, - /// on a q-value tie between charge states of a peptide, pick a different shared - /// (modseq, file) boundary. A no-work file already lands in this order today via + /// are already valid on disk). Applied to EVERY file the resume OVERLAYS, so the + /// WARM buffer order matches COLD regardless of whether the file was rescored -- + /// otherwise SecondPassFDR's BuildSharedBoundaries could iterate a different + /// order and, on a q-value tie between charge states of a peptide, pick a different + /// shared (modseq, file) boundary. A file the resume LOADS from its reconciled + /// parquet instead arrives in this order already, from + /// 's own canonical sort. A file with no + /// reconciliation work already lands in this order today via /// the single-key compaction sort (compacted EntryIds are unique per file), but /// sorting unconditionally future-proofs the tie-break against any later change /// that retains multiple rows per EntryId. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs index a14bba5d7d..77ca75555f 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileScoringTask.cs @@ -1345,9 +1345,9 @@ private FdrProjectionSet LoadJoinOnlyScores( // go lean" additionally excludes the reconciled-input merge. Do not "fix" this by // copying the builder decision: the merge does not read Features off these stubs - // both pass-2 shapes reload them per file from the reconciled parquet - // (ComputePass2TransferCompeteFull's own read, or ComputePass2Resident's), and - // EffectiveScoresPathFromScoresPath falls back to the original parquet when no - // reconciled one exists, so the reload does not depend on hasReconSidecars either. + // (ComputePass2TransferCompeteFull's own read, or ComputePass2Resident's), which + // Stage 6 writes for every run, so the reload does not depend on hasReconSidecars + // either. bool loadFeatures = needsResidentPool; // The --input-files paths at :381 and :644 THROW on the same O(files) situation. diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index 06cedd3af5..6a25d9e233 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -468,27 +468,88 @@ internal static bool StartsAfterPerFileScoring(OspreyConfig config) } /// - /// Each input's scores parquet, in input order: the reconciled sibling where Stage 6 - /// has written one, else the Stage 4 file. + /// Which per-run parquet THIS task reads its rows from: the Stage 6 + /// .scores-reconciled.parquet for SecondPassFDR, the Stage 4 + /// .scores.parquet for the two tasks that run before Stage 6 has written one. + /// + /// A property of the TASK, not of what happens to be on disk. It used to be + /// decided by probing for the reconciled sibling and taking it where it existed, + /// which gives the right answer only because the pipeline happens to run the stages + /// in order - the file is absent before Stage 6 and present after. Re-run + /// --task FirstPassFDR over a directory a previous run completed and the same + /// probe hands the FIRST pass the survivor SUBSET, roughly 1/52 of its rows, with + /// nothing to reject it: the version, search and library hashes all match. It then + /// writes cohort-wide boundary artifacts from that subset and exits 0. + /// + /// The two answers are also what an HPC node is SHIPPED. Boundary 3 -> 4 sends + /// a SecondPassFDR node the reconciled parquets and not the Stage 4 originals - + /// regression.ps1's mode-3 chain deletes them from the worker directory before + /// staging phase 4, so reaching for one fails on a missing file rather than passing + /// quietly. A FirstPassFDR or PerFileRescoring node gets the originals + /// and no reconciled sibling exists yet. So on a correct node each task has exactly + /// one of the two present, and asking the disk cannot distinguish "the artifact for + /// my pass" from "the only artifact here". + /// + internal static bool ReadsReconciledScores(OspreyConfig config) + { + return config.SelectedTask == HpcTask.SecondPassFdr; + } + + /// + /// True when THIS process runs Stage 7's join, i.e. when a per-run source published for + /// that join will actually be folded by something. + /// + /// Names what is ADMITTED, so it fails closed: the straight-through pipeline (no + /// --task, which runs every stage), the SecondPassFDR node, and + /// ModelDiagnostics - which is not an HPC fan-out node but does let + /// SecondPassFDR compute the pass-2 view, so it folds the same join and must not + /// be pushed back onto the resident pool. A task added later is excluded until someone + /// decides otherwise, which is the direction a predicate guarding a memory shape - and, + /// since , a correctness one - should + /// fail in. + /// + /// The excluded tasks each have a consumer that never arrives. + /// PerFileScoring and SpectraCache stop before Stage 5. + /// FirstPassFDR would publish empty per-run lists for a fold that never runs. + /// A PerFileRescoring worker exits after Stage 6, so a source built there is + /// never pulled - it only pays for a retained-sidecar read the "entering + /// PerFileRescoring must cost the same for 1 run as for 446" contract forbids, and + /// emits a streamed-join marker into a log for a join that did not happen. + /// + internal static bool RunsStage7Join(OspreyConfig config) + { + if (!config.SelectedTask.HasValue) + return true; + return config.SelectedTask == HpcTask.SecondPassFdr || + config.SelectedTask == HpcTask.ModelDiagnostics; + } + + /// + /// Each input's scores parquet for THIS task, in input order - see + /// for which one that is. Reached only by the + /// three tasks names, so every case has an + /// answer. /// /// The derivation --input-scores used to be handed ready-made. Its /// directory form globbed a directory and preferred the reconciled sibling per stem; - /// this is that rule, applied to the runs the command line names instead of to - /// whatever a directory happened to hold. The difference matters twice: a directory - /// with a stray parquet no longer changes the cohort, and ORDER is now the caller's - /// (FirstPassFDR reconciliation is order-sensitive, so a chain must pass a + /// this is the same list built from the runs the command line names instead of from + /// whatever a directory happened to hold. The difference matters three times: a + /// directory with a stray parquet no longer changes the cohort; ORDER is now the + /// caller's (FirstPassFDR reconciliation is order-sensitive, so a chain must pass a /// deterministically sorted list - which is what it already did to get a stable - /// directory sort). + /// directory sort); and the per-stem preference is no longer part of it. /// internal static List ScoresPathsForInputs(OspreyConfig config) { var paths = new List(config.InputFiles?.Count ?? 0); if (config.InputFiles == null) return paths; + bool reconciled = ReadsReconciledScores(config); foreach (string input in config.InputFiles) { - paths.Add(ParquetScoreCache.EffectiveScoresPathFromScoresPath( - ParquetScoreCache.GetScoresPath(input))); + paths.Add(reconciled + ? ParquetScoreCache.GetReconciledScoresPath(input) + : ParquetScoreCache.GetScoresPath(input)); } return paths; } @@ -554,10 +615,10 @@ internal static bool CanStreamStage7Join(OspreyConfig config, bool stage7Stream) /// TOP of Stage 6, hours before the rescore it is about to perform writes the parquets /// the full predicate asks about. Asking the full question there answers "no" on every /// cold run - not because the run cannot stream, but because it has not got there yet. - /// It does not need the term either: that arm rebuilds a run through - /// , which falls back to the Stage 4 parquet plus - /// the 1st-pass sidecar for a run with no reconciled sibling, where the reconciled-input - /// merge has nothing else to read. + /// It does not need the term either: by the time that arm's source is pulled, Stage 6 + /// has written a reconciled parquet for every run, so the question the full predicate + /// asks has an answer - and a run still missing one fails there rather than falling + /// back to its Stage 4 parquet. /// /// The retained base_id summary IS in this half even though it is an artifact: /// FirstPassFDR writes it before any caller of either form runs, so it is answerable on @@ -565,6 +626,18 @@ internal static bool CanStreamStage7Join(OspreyConfig config, bool stage7Stream) /// internal static bool Stage7StreamAdmittedBeforeRescore(OspreyConfig config, bool stage7Stream) { + // FIRST, and it is a correctness term rather than an optimisation. Every other term + // here describes the SHAPE of a Stage 7 join; none of them asks whether this process + // runs one. Without this, `--task FirstPassFDR` re-run over a COMPLETED directory + // satisfies all of them - the reconciled parquets are present because a previous + // pass wrote them - and PerFileScoringTask's `perRunJoin` branch then publishes one + // EMPTY list per run for a fold that never comes. FirstPassFDR computes its pass over + // nothing and rewrites both boundary sidecars and the retained base_id summary as + // empty, exit 0. The `ExpectReconciledInput` term this predicate replaced made that + // unreachable for anything but `--task SecondPassFDR`, so the hole opened when the + // proxy went and nothing took over the question it had been answering incidentally. + if (!RunsStage7Join(config)) + return false; if (!stage7Stream) return false; if (PerFileScoringTask.NeedsResidentPool(config, OspreyEnvironment.UseFdrProjection)) @@ -667,6 +740,14 @@ internal static string Stage7ResidentGuardError( { return null; } + // OSPREY_STAGE6_STREAM_SURVIVORS=0 withholds the survivor loader, so the cold arm + // publishes no per-run source however this run answers - which makes the remedy + // below ("unset OSPREY_STAGE7_STREAM") unachievable, and refusing on it would demand + // a token for a choice the operator does not have. The A/B oracle that switch exists + // to provide asks for BOTH stages resident; this is the one combination where + // `streamingAvailable` is true and streaming is nonetheless unreachable. + if (!OspreyEnvironment.Stage6StreamSurvivors) + return null; // The SUPPLIED value is quoted, matching the two sibling guards: a stale or // misspelled token otherwise reads exactly like an unset one, and the operator // cannot tell "you named nothing" from "you named the wrong path". diff --git a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs index 89e80ce17c..a2b359bd73 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/SecondPassFdrTask.cs @@ -72,13 +72,13 @@ public override bool IsIncluded(PipelineContext ctx) public override IEnumerable Inputs(PipelineContext ctx) { if (ctx.Config.InputFiles == null) yield break; - // Stage 7 reads the reconciled parquet when Stage 6 produced one, - // else the original Stage 4 parquet (no-work files). Recorded for - // provenance only -- the driver validates tasks by output sidecar + // Stage 7 reads the reconciled parquet, which Stage 6 writes for every run - + // a run without one fails in UnusableReconciledParquets rather than being read + // from its Stage 4 file, which a SecondPassFDR node is not even shipped. + // Recorded for provenance only -- the driver validates tasks by output sidecar // key, never by re-checking Inputs() existence (TaskValiditySidecar). foreach (var input in ctx.Config.InputFiles) - yield return ParquetScoreCache.EffectiveScoresPathFromScoresPath( - ParquetScoreCache.GetScoresPath(input)); + yield return ParquetScoreCache.GetReconciledScoresPath(input); // Under the frozen modes the per-run 2nd-pass FDR sidecars are this task's INPUTS: // the rescore worker computed and wrote them, and the join folds the per-base_id @@ -287,9 +287,17 @@ public override bool Run(PipelineContext ctx) // reachable with no token at all, which is the one shape the named-token ratchet is // supposed to make impossible. (The paragraph sat above the marker wipe below, far // from the call it describes.) - bool couldStream = ScoringTaskShared.CanStreamStage7Join(ctx.Config, stage7Stream: true); - string residentError = ScoringTaskShared.Stage7ResidentGuardError( - couldStream, OspreyEnvironment.Stage7Stream, OspreyEnvironment.AllowUnfixedResident); + // + // The switch is tested FIRST because `couldStream` is not free: its + // AllReconciledParquetsCurrent term opens a footer per run, and the guard's own + // first line discards the answer whenever the switch is on - so on the default + // path that was an O(files) sweep of a network artifact directory computed only to + // be thrown away. Nothing else here needs it. + string residentError = OspreyEnvironment.Stage7Stream + ? null + : ScoringTaskShared.Stage7ResidentGuardError( + ScoringTaskShared.CanStreamStage7Join(ctx.Config, stage7Stream: true), + OspreyEnvironment.Stage7Stream, OspreyEnvironment.AllowUnfixedResident); if (residentError != null) throw new InvalidOperationException(residentError); @@ -375,8 +383,19 @@ public override bool Run(PipelineContext ctx) // refuses outright no longer pays for 289 M survivors it is about to discard. // Its own transients are footer metadata, which is why it can sit between the // stage7-inherited and stage7-pool probes without distorting either. - var stale = StaleReconciledParquets(rescored.FileNames, perFileParquetPaths); - if (stale.Count > 0) + var unusable = UnusableReconciledParquets(rescored.FileNames, perFileParquetPaths); + if (unusable.Missing.Count > 0) + { + throw new InvalidOperationException(string.Format( + "{0} of {1} run(s) have no .scores-reconciled.parquet. Stage 6 writes one " + + "for every run, so these were not persisted - the write no-opped, failed, " + + "or the artifacts were not shipped to this node. Their .scores.parquet is " + + "not a substitute: it holds 1st-pass boundaries and none of the gap-fill " + + "rows. Re-run Stage 6 for them. Missing: [{2}].", + unusable.Missing.Count, rescored.FileCount, + string.Join(", ", unusable.Missing))); + } + if (unusable.Stale.Count > 0) { throw new InvalidOperationException(string.Format( "{0} of {1} reconciled parquet(s) predate the survivor-subset format, so " + @@ -385,7 +404,7 @@ public override bool Run(PipelineContext ctx) "unusable, so a parquet-only rewrite would leave the directory " + "inconsistent. Re-run the analysis from Stage 5 over this directory. " + "Stale: [{2}].", - stale.Count, rescored.FileCount, string.Join(", ", stale))); + unusable.Stale.Count, rescored.FileCount, string.Join(", ", unusable.Stale))); } // NO .Value here any more (#4486). Every consumer below folds through @@ -1076,27 +1095,40 @@ private static bool AnyReconciledParquet(OspreyConfig config) } /// - /// The per-file keys whose .scores-reconciled.parquet is on disk but predates - /// the survivor-subset format, so Stage 7 cannot read it. + /// The per-file keys whose .scores-reconciled.parquet Stage 7 cannot read, + /// split by WHY, because the two have different remedies: Missing means Stage 6 + /// never persisted the run, and Stale means what it wrote predates the + /// survivor-subset format. /// - /// The run refuses rather than converting: this branch changed the FDR - /// sidecars too, so an old directory has no self-consistent artifact set to + /// Absence is reported, not skipped. Stage 6 writes this artifact for every run - + /// WriteUnchangedReconciled covers the run it did no work on - so P13 makes + /// absence unambiguous at the producer, and this is the consumer end of the same + /// principle. Exempting a missing file here let a run whose write never landed pass + /// this gate and be silently rebuilt from its Stage 4 parquet, i.e. at 1st-pass + /// boundaries with no gap-fill rows, in a run that exits 0. + /// + /// The run refuses rather than converting a stale one: this branch changed the + /// FDR sidecars too, so an old directory has no self-consistent artifact set to /// convert toward and has to be re-run from Stage 5 (issue #4486). /// - private static List StaleReconciledParquets( + private static (List Missing, List Stale) UnusableReconciledParquets( IReadOnlyList fileNames, IReadOnlyDictionary perFileParquetPaths) { + var missing = new List(); var stale = new List(); if (perFileParquetPaths == null) - return stale; + return (missing, stale); foreach (var fileName in fileNames) { if (!perFileParquetPaths.TryGetValue(fileName, out string scoresPath)) continue; string reconciledPath = ParquetScoreCache.ReconciledPathFromScoresPath(scoresPath); if (!File.Exists(reconciledPath)) + { + missing.Add(fileName); continue; + } // Stale is EITHER an older generation (marker mismatch) OR the interim // #4486 shape - survivor subset with no score_index column - which the // per-file loaders would otherwise read by POSITION, silently binding @@ -1112,7 +1144,7 @@ private static List StaleReconciledParquets( if (!ParquetScoreCache.IsCurrentReconciledSurvivorSubset(reconciledPath)) stale.Add(fileName); } - return stale; + return (missing, stale); } /// /// Write passing entries to a BiblioSpec blib file. diff --git a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs index 239fb7a34c..fad55fc1a5 100644 --- a/pwiz_tools/Osprey/Osprey.Test/IOTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/IOTest.cs @@ -2227,35 +2227,40 @@ public void TestReconciledNamingUnambiguousForReconciledStem() } /// - /// Verifies EffectiveScoresPathFromScoresPath returns the reconciled - /// sibling when it exists on disk, else the original -- the per-file - /// read contract that makes the separate-reconciled-file design - /// byte-equivalent to the former in-place overwrite. + /// Which parquet a task reads is decided by the TASK, and NOT by what is on disk. + /// + /// The predecessor of this test pinned the opposite - a probe that took the + /// reconciled sibling wherever it existed. That gives the right answer only because + /// the pipeline runs the stages in order, and it silently gives the wrong one on a + /// re-run: --task FirstPassFDR over a directory a previous run completed found + /// the reconciled parquets and would have trained the FIRST pass on the survivor + /// SUBSET, with every version, search and library hash matching. So the assertion + /// that matters is the negative one - BOTH files present, and the task still decides. + /// A test that laid down only one file would pass against the probe as well. /// [TestMethod] - public void TestEffectiveScoresPathFromScoresPath() + public void TestScoresPathsDependOnTaskNotDisk() { string dir = Path.Combine(Path.GetTempPath(), "osprey_eff_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); try { + string input = Path.Combine(dir, "sample1.mzML"); string original = Path.Combine(dir, "sample1.scores.parquet"); string reconciled = Path.Combine(dir, "sample1.scores-reconciled.parquet"); + // BOTH on disk, which is the state a completed run leaves behind. + File.WriteAllText(input, "x"); File.WriteAllText(original, "x"); - - // No reconciled sibling -> original (no-work file). - Assert.AreEqual(original, - ParquetScoreCache.EffectiveScoresPathFromScoresPath(original)); - - // Reconciled sibling present -> reconciled (rescored file). File.WriteAllText(reconciled, "y"); - Assert.AreEqual(reconciled, - ParquetScoreCache.EffectiveScoresPathFromScoresPath(original)); - // An already-reconciled input that exists is returned as-is. - Assert.AreEqual(reconciled, - ParquetScoreCache.EffectiveScoresPathFromScoresPath(reconciled)); + // The two passes that run BEFORE Stage 6 read the Stage 4 file even though + // the reconciled sibling is sitting beside it. + AssertScoresPathForTask(input, HpcTask.FirstPassFdr, original); + AssertScoresPathForTask(input, HpcTask.PerFileRescore, original); + + // The join reads the reconciled one - the only artifact its node is shipped. + AssertScoresPathForTask(input, HpcTask.SecondPassFdr, reconciled); } finally { @@ -2263,6 +2268,22 @@ public void TestEffectiveScoresPathFromScoresPath() } } + /// + /// One input, one task, one expected parquet - through the same helper the pipeline + /// calls, so the test cannot agree with a rule the tasks do not use. + /// + private static void AssertScoresPathForTask(string input, HpcTask task, string expected) + { + var config = new OspreyConfig + { + SelectedTask = task, + InputFiles = new List { input } + }; + var paths = ScoringTaskShared.ScoresPathsForInputs(config); + Assert.AreEqual(1, paths.Count); + Assert.AreEqual(expected, paths[0]); + } + /// /// Verifies that writing an empty list does not create a file. /// @@ -2848,10 +2869,18 @@ public void TestIsCurrentReconciledSurvivorSubset() // Written the way Stage 6 writes it - the marker in the footer, score_index in // the schema - which is the only combination that passes. - var metadata = new Dictionary - { - { "osprey.reconciled", ParquetScoreCache.RECONCILED_SURVIVORS } - }; + // + // Metadata from the REAL producer, not hand-fed. StreamReconciledScoresParquet + // writes the caller's map verbatim, so a test that supplies the marker itself + // asserts only that the reader can read what the test wrote: let + // ReconciledParquetWriter stop stamping it and this would still pass while + // AllReconciledParquetsCurrent returned false for every real run and the whole + // cohort fell back to the resident join. Going through the producer is what + // makes the two sides able to disagree. + var metadata = ReconciledParquetWriter.BuildReconciliationMetadata( + new OspreyConfig(), null); + Assert.AreEqual(ParquetScoreCache.RECONCILED_SURVIVORS, + metadata["osprey.reconciled"]); ParquetScoreCache.StreamReconciledScoresParquet( originalPath, reconciledPath, null, null, metadata, null, "f.mzML", null, null, null); diff --git a/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs b/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs index 3d5006d548..c6081185f2 100644 --- a/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/LibraryFragmentReleaseTest.cs @@ -282,7 +282,9 @@ private static OspreyConfig ForTask(HpcTask task) { SelectedTask = task, NoJoin = task == HpcTask.PerFileScoring || task == HpcTask.PerFileRescore, - StopAfterStage5 = task == HpcTask.FirstPassFdr || task == HpcTask.ModelDiagnostics, + // EXACTLY Program.cs's single assignment. Naming ModelDiagnostics here built + // a config the CLI cannot produce - see PipelineMembershipTest.ForTask. + StopAfterStage5 = task == HpcTask.FirstPassFdr, ExpectReconciledInput = task == HpcTask.SecondPassFdr, }; } diff --git a/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs b/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs index 000457c1c9..e8737003a3 100644 --- a/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs +++ b/pwiz_tools/Osprey/Osprey.Test/PipelineMembershipTest.cs @@ -21,6 +21,8 @@ * limitations under the License. */ +using System; +using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using pwiz.Osprey.Core; using pwiz.Osprey.Tasks; @@ -56,7 +58,14 @@ private static OspreyConfig ForTask(HpcTask task) { SelectedTask = task, NoJoin = task == HpcTask.PerFileScoring || task == HpcTask.PerFileRescore, - StopAfterStage5 = task == HpcTask.FirstPassFdr || task == HpcTask.ModelDiagnostics, + // EXACTLY Program.cs's assignment, which is the only one in the tree: + // `config.StopAfterStage5 = selectedTask == HpcTask.FirstPassFdr;`. This + // helper also named ModelDiagnostics, building a config the CLI cannot + // produce - so the row below asserted a membership no real run has, while + // ProgramTests pinned the real flags and stated the opposite design. Two + // tests in one assembly asserting incompatible things is worse than either + // being wrong alone, because whichever you read first looks corroborated. + StopAfterStage5 = task == HpcTask.FirstPassFdr, ExpectReconciledInput = task == HpcTask.SecondPassFdr, }; } @@ -78,16 +87,15 @@ public void TestIsIncludedMembershipTable() new[] { false, false, true, false }), (@"SecondPassFDR", ForTask(HpcTask.SecondPassFdr), new[] { false, false, false, true }), - // --task ModelDiagnostics is a RENDER over retained products, not a stage. - // It needs the per-file load (so PerFileScoring is in) and first-pass state - // (so FirstPassFDR is), and nothing after: a diagnostics fold publishes - // neither CompactedEntries nor a second pass, and the two tasks that demand - // them used to join anyway and fail the run AFTER writing the report it was - // asked for. The row that stood here was `input-scores-full` - the - // single-node full pipeline started from parquets - and it retired with the - // flag; this is the mode that was actually at risk. + // --task ModelDiagnostics is a RENDER over retained products, and it reaches + // AnalysisPipeline with all three membership flags FALSE - it sets none of + // them (see ForTask, and Program.cs's single StopAfterStage5 assignment). So + // it is in every task, exactly like the straight-through run, and suppresses + // artifact writes rather than membership. The row here used to read + // {true,true,false,false}, which was the shape of a config the CLI cannot + // build; ProgramTests.cs pins the real flags and now agrees with this. (@"ModelDiagnostics", ForTask(HpcTask.ModelDiagnostics), - new[] { true, true, false, false }), + new[] { true, true, true, true }), }; foreach (var c in cases) @@ -104,5 +112,42 @@ public void TestIsIncludedMembershipTable() } } } + + /// + /// Only a process that RUNS Stage 7's join may be admitted to the streamed one. + /// + /// The case that matters is --task FirstPassFDR, and it is not + /// hypothetical: re-run over a directory a previous analysis COMPLETED, every + /// disk-side term of CanStreamStage7Join is satisfied by that previous run's + /// own output. PerFileScoringTask would then take its per-run-join branch, + /// publish one EMPTY list per run for a fold that never comes, and FirstPassFDR would + /// compute its pass over nothing - rewriting both boundary sidecars and the retained + /// base_id summary as empty, exit 0. + /// + /// Asserted with the switch passed as TRUE and against the two-argument form, so + /// this pins the membership term alone and cannot pass merely because the environment + /// happens to have streaming off. + /// + [TestMethod] + public void TestOnlyStage7JoinTasksAdmitTheStreamedJoin() + { + var admitted = new[] { HpcTask.SecondPassFdr, HpcTask.ModelDiagnostics }; + foreach (HpcTask task in Enum.GetValues(typeof(HpcTask))) + { + bool expected = admitted.Contains(task); + Assert.AreEqual(expected, ScoringTaskShared.RunsStage7Join(ForTask(task)), + string.Format(@"--task {0}: RunsStage7Join must be {1}", task, expected)); + // A task that does not run the join must be refused BEFORE any disk term, + // which is what makes the refusal free and unconditional. + if (!expected) + { + Assert.IsFalse( + ScoringTaskShared.Stage7StreamAdmittedBeforeRescore(ForTask(task), true), + string.Format(@"--task {0} must not be admitted to the streamed join", task)); + } + } + // The straight-through pipeline runs every stage, so it is admitted. + Assert.IsTrue(ScoringTaskShared.RunsStage7Join(new OspreyConfig())); + } } } diff --git a/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs b/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs index 8aaefa932f..31056603c0 100644 --- a/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs +++ b/pwiz_tools/Osprey/Osprey.Test/ProgramTests.cs @@ -24,7 +24,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.IO; using Microsoft.VisualStudio.TestTools.UnitTesting; using pwiz.Osprey.Core; using pwiz.Osprey.IO; @@ -68,6 +67,40 @@ public void RestoreExperimentAgg() OspreyEnvironment.MeanBestN = _savedMeanBestN; } + /// + /// Two inputs sharing a file-name STEM are refused, whatever directories they sit in. + /// + /// Every per-run artifact is <stem>.<suffix> and every per-run + /// map is keyed the same way, so a shared stem is two runs the pipeline cannot tell + /// apart. Left to be discovered downstream it takes two shapes and neither names the + /// cause: an ArgumentException about a duplicate key mid-Stage-6/7, or - with + /// --output-dir, where both stems resolve into one directory - two runs quietly + /// sharing one parquet, no error at all. + /// + /// Asserted with DIFFERENT directories, which is the case that matters and the + /// one --input-list makes routine at cohort scale; identical paths would be + /// caught by cruder means. + /// + [TestMethod] + public void TestValidateRejectsDuplicateInputStems() + { + var config = TaskConfig(HpcTask.PerFileScoring); + config.LibrarySource = LibrarySource.FromPath("ref.blib"); + config.InputFiles = new List { @"plateA\run1.mzML", @"plateB\run1.mzML" }; + string err = Program.ValidateArgs(config); + Assert.IsNotNull(err, "two inputs sharing a stem must be refused"); + // The stem and BOTH colliding paths, so the operator can act without re-deriving + // which of several hundred inputs collided. + StringAssert.Contains(err, "run1"); + StringAssert.Contains(err, @"plateA\run1.mzML"); + StringAssert.Contains(err, @"plateB\run1.mzML"); + + // Distinct stems in one directory remain fine - the check is on the stem, not the + // directory, and a cohort in one folder is the ordinary case. + config.InputFiles = new List { @"plateA\run1.mzML", @"plateA\run2.mzML" }; + Assert.IsNull(Program.ValidateArgs(config)); + } + // --- ValidateArgs: what each task requires ------------------------- private static OspreyConfig TaskConfig(HpcTask task) @@ -837,14 +870,5 @@ public void TestParseArgsDecoysInLibraryDefaultsFalse() Assert.IsTrue(string.IsNullOrEmpty(config.DecoyPairingManifestPath)); } - // --- helpers ------------------------------------------------------- - - private static string NewTempDir() - { - string dir = Path.Combine(Path.GetTempPath(), - "osprey_test_program_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(dir); - return dir; - } } } diff --git a/pwiz_tools/Osprey/Osprey/Program.cs b/pwiz_tools/Osprey/Osprey/Program.cs index d38d7bd98a..64b9b00762 100644 --- a/pwiz_tools/Osprey/Osprey/Program.cs +++ b/pwiz_tools/Osprey/Osprey/Program.cs @@ -22,10 +22,14 @@ */ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using pwiz.Common.SystemUtil; using pwiz.Osprey.Core; using pwiz.Osprey.IO; +using pwiz.Osprey.Tasks; using pwiz.Osprey.Tasks.ModelDiagnostics; namespace pwiz.Osprey @@ -223,8 +227,23 @@ static int Main(string[] args) // would refuse the configuration the HPC chain is built on. This is // what --input-scores used to say by naming a different input KIND; // said here it is one input kind and one question about it. - if (File.Exists(ParquetScoreCache.EffectiveScoresPathFromScoresPath( - ParquetScoreCache.GetScoresPath(inputFile)))) + // ...and only for a task that STARTS AFTER Stage 4. A scores parquet + // stands an input in because such a task never opens the data file; it + // stands in for nothing at all for --task SpectraCache or PerFileScoring, + // whose whole product is decoded FROM that file. Without this term a + // mistyped or moved input on those tasks proceeds on a leftover parquet + // and logs that the run will be read from its scores parquet, "which is + // what a task after Stage 4 needs" - false for exactly the two tasks that + // could reach it. The old `if (!fromInputScores)` wrapper could not reach + // them structurally; nothing re-established that scoping when it went. + // + // EITHER parquet, named. WHICH one a task reads is that task's question + // (ScoringTaskShared.ReadsReconciledScores) and this runs before dispatch: + // a FirstPassFDR node is shipped .scores.parquet, a SecondPassFDR + // node only .scores-reconciled.parquet. + if (ScoringTaskShared.StartsAfterPerFileScoring(config) && + (File.Exists(ParquetScoreCache.GetScoresPath(inputFile)) || + File.Exists(ParquetScoreCache.GetReconciledScoresPath(inputFile)))) { artifactOnlyInputs++; continue; @@ -539,6 +558,38 @@ private static string TaskCliName(HpcTask task) /// failure. Does not log warnings (those stay in ). Internal so /// Osprey.Test can exercise it. /// + /// + /// An error naming every input stem that appears more than once, with the paths that + /// collide, or null when all stems are distinct. Ordinal comparison, matching the + /// per-run maps this protects. + /// + private static string DuplicateInputStemError(IReadOnlyList inputFiles) + { + var byStem = new Dictionary>(StringComparer.Ordinal); + foreach (string input in inputFiles) + { + string stem = Path.GetFileNameWithoutExtension(input) ?? string.Empty; + if (!byStem.TryGetValue(stem, out var paths)) + { + paths = new List(); + byStem[stem] = paths; + } + paths.Add(input); + } + var collisions = byStem.Where(kv => kv.Value.Count > 1).ToList(); + if (collisions.Count == 0) + return null; + var sb = new StringBuilder(); + sb.AppendFormat( + "{0} input stem(s) appear more than once. Every per-run artifact is named " + + "., so runs sharing a stem cannot be told apart and would " + + "overwrite each other's parquets and sidecars. Rename or stage them so each " + + "run has a distinct file name:", collisions.Count); + foreach (var kv in collisions) + sb.AppendFormat("\n '{0}': {1}", kv.Key, string.Join(", ", kv.Value)); + return sb.ToString(); + } + internal static string ValidateArgs(OspreyConfig config) { bool hasInputFiles = config.InputFiles != null && config.InputFiles.Count > 0; @@ -552,6 +603,27 @@ internal static string ValidateArgs(OspreyConfig config) if (aggErr != null) return aggErr; + // Every run is keyed on its input STEM - the per-file artifacts are + // ., and every per-run map in the pipeline is keyed the same way - + // so two inputs sharing a stem are two runs the pipeline cannot tell apart. It is + // not exotic: --input-list makes it routine at cohort scale, where the same + // acquisition name recurs under different directories. + // + // Refused here rather than surviving to be discovered downstream, where it takes + // two shapes and neither says what happened. Without --output-dir the join appends + // two rows under one key while the parquet map keeps only the second, and + // CurrentReconciledPaths dies with "An item with the same key has already been + // added" mid-Stage-6/7. WITH --output-dir it is worse and silent: both stems + // resolve into the same directory, so the two runs share one .scores.parquet and + // one .scores-reconciled.parquet, each overwriting the other, with no error at all. + // The retired --input-scores form made stems unique by construction. + if (hasInputFiles) + { + string dupErr = DuplicateInputStemError(config.InputFiles); + if (dupErr != null) + return dupErr; + } + if (config.SelectedTask.HasValue) { switch (config.SelectedTask.Value) diff --git a/pwiz_tools/Osprey/README.md b/pwiz_tools/Osprey/README.md index deb218fc61..2bb2e9f2de 100644 --- a/pwiz_tools/Osprey/README.md +++ b/pwiz_tools/Osprey/README.md @@ -109,9 +109,16 @@ fan-out boundaries into four single-task workers — one node = one | `--task` | shape | reads | writes (next to the input) | |----------|-------|-------|-----------------------------| | `PerFileScoring` | split 1 — per file | mzML (`-i`) + library (`-l`) | `.scores.parquet`, `.calibration.json` | -| `FirstPassFDR` | join 1 — all files | every `.scores.parquet` (`--input-scores`) | `.1st-pass.fdr_scores.bin`, `.reconciliation.json` | +| `FirstPassFDR` | join 1 — all files | every `.scores.parquet` | `.1st-pass.fdr_scores.bin`, `.reconciliation.json` | | `PerFileRescoring` | split 2 — per file | `.scores.parquet` + co-located `.1st-pass.fdr_scores.bin`, `.reconciliation.json` | `.scores-reconciled.parquet` | -| `SecondPassFDR` | join 2 — all files | every `.scores-reconciled.parquet` (`--input-scores`) | `.blib` (+ `.2nd-pass.fdr_scores.bin` when protein FDR is on) | +| `SecondPassFDR` | join 2 — all files | every `.scores-reconciled.parquet` | `.blib` (+ `.2nd-pass.fdr_scores.bin` when protein FDR is on) | + +Every task names its runs with `-i` / `--input-list`, giving the **data files**, and derives +each run's parquet and sidecars from the input stem plus `--output-dir`. Which parquet a task +reads is a property of the task — `FirstPassFDR` and `PerFileRescoring` read `.scores.parquet`, +`SecondPassFDR` reads `.scores-reconciled.parquet` — not of what happens to be in the +directory. The data file itself need not still exist: a node whose `.spectra.bin` or scores +parquet is staged is accepted without it. The driver also writes a `..osprey.task` validity sidecar next to each output; re-running a task whose outputs already exist @@ -128,17 +135,22 @@ Osprey -i *.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 Osprey --task PerFileScoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 # -> s1.scores.parquet, s1.calibration.json (next to s1.mzML; -o is ignored here) -# Join 1 — FirstPassFDR, one process over ALL parquets (pass a DIRECTORY so order is fixed): -Osprey --task FirstPassFDR --input-scores ./scores_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 +# Join 1 — FirstPassFDR, one process over ALL runs (name them in a fixed order): +Osprey --task FirstPassFDR -i s1.mzML -i s2.mzML -i s3.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 +# reads s1.scores.parquet, s2..., s3... # -> .1st-pass.fdr_scores.bin, .reconciliation.json (next to each parquet) # Split 2 — PerFileRescoring, one process per file (parquet + its two sidecars co-located): -Osprey --task PerFileRescoring --input-scores s1.scores.parquet -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 +Osprey --task PerFileRescoring -i s1.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 # -> s1.scores-reconciled.parquet -# Join 2 — SecondPassFDR, one process over ALL reconciled parquets (DIRECTORY again): -Osprey --task SecondPassFDR --input-scores ./reconciled_dir -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 -# -> out.blib +# Join 2 — SecondPassFDR, one process over ALL runs (same order): +Osprey --task SecondPassFDR -i s1.mzML -i s2.mzML -i s3.mzML -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 +# reads s1.scores-reconciled.parquet, s2..., s3... -> out.blib + +# Past a few hundred runs, --input-list takes one path per line and composes with -i +# (446 -i paths measured ~28,600 characters against a 32,767 command-line limit): +Osprey --task SecondPassFDR --input-list runs.txt -l hela.tsv -o out.blib --resolution unit --protein-fdr 0.01 ``` The example above shows the *commands*. What each node must actually be **shipped** at each @@ -151,14 +163,16 @@ often proceed without them and produce a plausible wrong answer - is the relay c - **Same parameters on every task.** Pass an identical `-l ` and identical search flags (`--resolution`, `--protein-fdr`, ...) to all four tasks. The parquet integrity check (`osprey.search_hash` footer - metadata) rejects `--input-scores` files whose search/library hash does - not match the current invocation. -- **`--input-scores` ordering is significant.** A *directory* argument is - globbed and sorted internally (deterministic). An explicit *file list* - is consumed in the order given. FirstPassFDR reconciliation is - order-sensitive, so for `FirstPassFDR` and `SecondPassFDR` pass a directory or a + metadata) rejects parquets whose search/library hash does not match the + current invocation. +- **Input ORDER is significant, and it is now yours.** Runs are consumed in + the order given to `-i` / `--input-list`. FirstPassFDR reconciliation is + order-sensitive, so for `FirstPassFDR` and `SecondPassFDR` pass a deterministically sorted list — a workflow engine's channel order is - otherwise nondeterministic and would cause run-to-run drift. + otherwise nondeterministic and would cause run-to-run drift. The retired + `--input-scores` sorted a globbed directory on your behalf; naming the runs + means the order is stated rather than inherited from a directory listing, + and a stray parquet in that directory can no longer change the cohort. - **Outputs land next to inputs; sidecars travel with the parquet.** Each task writes its outputs and sidecars beside the input file (not into cwd or a separate output dir). `PerFileRescoring` rehydrates from @@ -171,7 +185,7 @@ often proceed without them and produce a plausible wrong answer - is the relay c several files concurrently in one process) and would double-parallelize under a scheduler. - **`--help` is the authoritative flag reference** (`Osprey --help`), - with a Distributed / HPC group covering `--task` and `--input-scores`. + with a Distributed / HPC group covering `--task`, `-i` and `--input-list`. - **Exit codes**: a failing task returns a non-zero process exit code, so a workflow engine can gate on it normally. diff --git a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md index 46d0186fd2..0c6c1ece6c 100644 --- a/pwiz_tools/Osprey/docs/00-pipeline-architecture.md +++ b/pwiz_tools/Osprey/docs/00-pipeline-architecture.md @@ -621,12 +621,12 @@ node running that task needs a copy, whatever batch it was handed. | `.libcache` | experiment cache | library load, any task | all tasks | rebuild locally | | `.spectra.bin` | per-run cache | `PerFileScoring` (Stage 2), or `--task SpectraCache` | `PerFileScoring`, `PerFileRescoring` | with the run | | `.calibration.json` | per-run product | `PerFileScoring` (Stage 3) | `PerFileScoring`, `PerFileRescoring`, `FirstPassFDR`, `SecondPassFDR` | with the run, on **every** leg | -| `.scores.parquet` | per-run product | `PerFileScoring` (Stage 4) | `FirstPassFDR`, `PerFileRescoring`, **`SecondPassFDR`** (fallback for runs with no reconciled sibling) | with the run | +| `.scores.parquet` | per-run product | `PerFileScoring` (Stage 4) | `FirstPassFDR`, `PerFileRescoring` | with the run | | `.1st-pass.fdr_scores.bin` | per-run product | `FirstPassFDR` (pass 1) | `PerFileRescoring`; `SecondPassFDR` only under `OSPREY_PASS2_VERIFY_WORKER` or where no worker answer exists | with the run | | `.reconciliation.json` | per-run product | `FirstPassFDR` (Stage 6 planning) | `PerFileRescoring`, `SecondPassFDR` (gap-fill entry ids) | with the run | | `.1st-pass.fdr_experiment.bin` | experiment product | `FirstPassFDR` | `PerFileRescoring`, `SecondPassFDR`, `PerFileScoring` (rehydrate) | **every node** | | `.1st-pass.model.json` | experiment product, replicated | `FirstPassFDR` (training) | `PerFileRescoring`, `SecondPassFDR` | **every node** (any one copy) | -| `.scores-reconciled.parquet` | per-run product | `PerFileRescoring` (Stage 6) | `SecondPassFDR`, and `PerFileRescoring` itself on its per-run resume arm | with the run | +| `.scores-reconciled.parquet` | per-run product, written for **every** run | `PerFileRescoring` (Stage 6) | `SecondPassFDR` - the join's only row source, one parquet per run; and `PerFileRescoring` itself on its per-run resume arm | with the run | | `.2nd-pass.fdr_decoys.bin` | per-run product | `PerFileRescoring` (pass-2 worker) | `SecondPassFDR` | with the run | | `.2nd-pass.fdr_scores.bin` | per-run product | `PerFileRescoring` (pass-2 worker), else `SecondPassFDR` | `SecondPassFDR` | with the run | | `.2nd-pass.fdr_experiment.bin` | experiment product | `SecondPassFDR` | `SecondPassFDR` on a resume | n/a | @@ -1050,6 +1050,16 @@ inputs. The run log says which shape it took - "folding over N run(s), each rebu own artifacts and dropped" - and that line is the evidence, because a resident pool and a fold produce identical output and differ only in a memory profile. +**One parquet per run, and it is the reconciled one.** `.scores.parquet` is not an +input to this boundary in any form - not as a fallback, not for a run Stage 6 did no work on. +Stage 6 writes a reconciled parquet for *every* run (P13; `WriteUnchangedReconciled` covers +the no-work run), so a missing one means the write never landed and the run is not finished. +Substituting the Stage 4 file would put 1st-pass boundaries and no gap-fill rows into the +blib for that run from a process that exits 0, which is exactly the ambiguity P13 exists to +remove - so every consumer here **fails** on absence instead. The rule survived one earlier +round as "read the reconciled parquet, Stage 4's only as the per-file fallback"; the fallback +half is retired, and the code carries no path to it. + Not needed on the default path: `.1st-pass.fdr_scores.bin`. Establishing that is what issue #4486 was for - an orchestrator hands a `SecondPassFDR` node the per-run second-pass artifacts and the analysis-wide experiment sidecar, and nothing per-run from diff --git a/pwiz_tools/Osprey/docs/14-intermediate-files.md b/pwiz_tools/Osprey/docs/14-intermediate-files.md index ae3ecb1fdf..fde3284581 100644 --- a/pwiz_tools/Osprey/docs/14-intermediate-files.md +++ b/pwiz_tools/Osprey/docs/14-intermediate-files.md @@ -312,11 +312,18 @@ Stage 6 (`PerFileRescoreTask`) writes `.scores-reconciled.parquet` (`GetReconciledScoresPath`, `ParquetScoreCache.cs:1055`) rather than overwriting the Stage 4 `.scores.parquet`. The `.scores-reconciled.parquet` suffix is appended **after** the `.scores` token so it is an unambiguous "Stage 6 output" signal (`ParquetScoreCache.cs:1036`). -`EffectiveScoresPathFromScoresPath` (`ParquetScoreCache.cs:1103`) is the read-side contract: a -post-Stage-6 reader consumes the reconciled sibling when it exists on disk, else the original — -making the split-file design byte-equivalent to the former in-place overwrite while surviving a -partial Stage 6 crash. This is a C# infrastructure refinement over the Rust doc's single-file -model. +The read-side contract is `ScoringTaskShared.ReadsReconciledScores`: which of the two a reader +consumes is decided by the **task**, not by which file happens to be on disk. `SecondPassFDR` +reads the reconciled parquet; `FirstPassFDR` and `PerFileRescoring` read the Stage 4 file. +This is a C# infrastructure refinement over the Rust doc's single-file model, and it survives a +partial Stage 6 crash. + +It was a disk probe until 2026-09-08 - take the reconciled sibling wherever it exists, else the +original - which reads as equivalent and is not. It is right only while the stages run in order, +because the artifact is absent before Stage 6 and present after; re-run `--task FirstPassFDR` +over a completed directory and the same probe hands the FIRST pass the survivor SUBSET, with +every version, search and library hash matching. The task always knew which artifact it wanted; +the probe was inferring it from a side effect. --- @@ -667,9 +674,9 @@ default resume mechanism. - **[INTENTIONAL-CSHARP-DESIGN] Reconciled parquet is a separate `.scores-reconciled.parquet`, not an in-place overwrite** - Rust doc's model rewrites `.scores.parquet` in place during Stage - 6; C# writes a distinct sibling and selects it on read via - `EffectiveScoresPathFromScoresPath`, surviving a partial Stage 6 crash. Evidence: - `ParquetScoreCache.cs:1036,1055,1103`; `ReconciledParquetWriter.cs`. Severity: minor. + 6; C# writes a distinct sibling and selects it on read by task membership + (`ScoringTaskShared.ReadsReconciledScores`), surviving a partial Stage 6 crash. Evidence: + `ParquetScoreCache.cs`; `ReconciledParquetWriter.cs`. Severity: minor. - **[INTENTIONAL-CSHARP-DESIGN] FDR sidecar loader matches records by `entry_id`, tolerating `count < entries.len()`** - Rust doc says `entry_count` must equal `entries.len()` and records diff --git a/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md b/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md index 24e533a048..1157b27a32 100644 --- a/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md +++ b/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md @@ -55,9 +55,9 @@ The exact per-task membership per mode is pinned by `Osprey.Test/PipelineMembers | `--task FirstPassFDR` (`StopAfterStage5`) | rehydrate | run | – | – | | `--task PerFileRescoring` (`NoJoin`, `SelectedTask`) | rehydrate | rehydrate | run | – | | `--task SecondPassFDR` (`ExpectReconciledInput`) | rehydrate | (skipped) | rehydrate | run | -| `--task ModelDiagnostics` (`StopAfterStage5`) | rehydrate | rehydrate | – | – | +| `--task ModelDiagnostics` (no flag - sets NONE of the three) | run | run | run | run | -("rehydrate" = excluded from the driver loop but lazily materialized on demand from disk; "–" = never touched.) The predicates live in `PerFileScoringTask.IsIncluded` (`:84-88`), `FirstPassFdrTask.IsIncluded` (`:80-95`), `PerFileRescoreTask.IsIncluded` (`:123-130`), and `SecondPassFdrTask.IsIncluded` (`:57-64`). +("rehydrate" = excluded from the driver loop but lazily materialized on demand from disk; "–" = never touched.) `--task ModelDiagnostics` sets none of the three flags — `StopAfterStage5` has exactly one assignment in the tree and it names `FirstPassFdr` alone — so it is a member of every task, like the straight-through run, and suppresses artifact WRITES rather than membership. It is listed here because a truth-table row claiming otherwise stood in this file and in a unit test. The predicates live in `PerFileScoringTask.IsIncluded` (`:84-88`), `FirstPassFdrTask.IsIncluded` (`:80-95`), `PerFileRescoreTask.IsIncluded` (`:123-130`), and `SecondPassFdrTask.IsIncluded` (`:57-64`). ## Stage 1-4 — Per-file scoring (`--task PerFileScoring`) @@ -121,11 +121,11 @@ The row that used to sit beside it, `--input-scores` with no `--task` (a single- Every task takes `-i` / `--input-list`, naming the **data files**, and derives each run's parquet and sidecars from the input stem plus `--output-dir`. The data file need not still exist: `Program.Main`'s input check accepts a run whose `.spectra.bin` is on disk (delete-the-sources-after-caching), and also one whose `.scores.parquet` - or its reconciled sibling - is on disk, which is the state a staged worker directory is in. -- **Reconciled wins per stem**, as it always did: `ScoringTaskShared.ScoresPathsForInputs` resolves each input through `ParquetScoreCache.EffectiveScoresPathFromScoresPath`, so a run whose Stage 6 output exists is read from the reconciled parquet and one without it from the Stage 4 file. +- **The TASK picks the parquet, not the directory.** `ScoringTaskShared.ScoresPathsForInputs` resolves each input through `ReadsReconciledScores`: `SecondPassFDR` reads `.scores-reconciled.parquet`, `FirstPassFDR` and `PerFileRescoring` read `.scores.parquet`. That is also what each node is shipped (Boundary 3 -> 4), so on a correct node only one of the two is present - which is exactly why this must not be decided by looking. The rule was "reconciled wins per stem", inherited from `--input-scores`' directory form; it is right only while the stages run in order, and a `--task FirstPassFDR` re-run over a completed directory would take the survivor subset as its first-pass population. - **Order is the caller's.** FirstPassFDR reconciliation is order-sensitive, so a chain must pass a deterministically sorted list. `--input-scores` used to sort a globbed directory Ordinal on the caller's behalf; naming the runs explicitly means an orchestrator states the order rather than inheriting it from a directory listing, and a stray parquet in that directory can no longer change the cohort. - `--input-list` takes one path per line (blank lines and `#` comments ignored) and composes with `-i`. It is what a cohort past a few hundred runs needs: 446 `-i` paths measured ~28,600 characters against a 32,767 limit. -**Why the flag went.** It named an input KIND - "you handed me parquets" - which is how the Rust pipeline said *Stage 1-4 is done*. The C# port says that with `--task` plus the per-run validity sidecars, and two seams answering one question is what let `--task ModelDiagnostics` (which sets `StopAfterStage5`, a C#-era signal, while its inputs were mzML stems, a Rust-era one) join the pipeline and demand state a diagnostics fold never publishes. The clearest evidence it was a round trip: the pipeline's first act was `RescoreHydration.SyntheticInputFromParquet`, rebuilding a synthetic `.mzML` that does not exist, purely so the sidecar path helpers could work. +**Why the flag went.** It named an input KIND - "you handed me parquets" - which is how the Rust pipeline said *Stage 1-4 is done*. The C# port says that with `--task` plus the per-run validity sidecars, and two seams answering one question is what let `--task ModelDiagnostics` (a C#-era signal, while its inputs were mzML stems, a Rust-era one) join the pipeline and demand state a diagnostics fold never publishes. The clearest evidence it was a round trip: the pipeline's first act was `RescoreHydration.SyntheticInputFromParquet`, rebuilding a synthetic `.mzML` that does not exist, purely so the sidecar path helpers could work. ## Parquet footer hash validation @@ -201,7 +201,7 @@ a corrupt cache a downstream stage must reject. See principle P8 in - **[INTENTIONAL-CSHARP-DESIGN] One name per task, describing the FDR pass** - The CLI name, the `HpcTask` member, the task class, the `[TASK]` log token, and the `.osprey.task` stamp are all one string per task, describing the FDR pass rather than the join topology. Two of them used to describe the topology instead (`FirstJoinTask`/`FirstPassFDR` and `MergeNodeTask`/`SecondPassFDR`), which cost a reader a mapping table and once produced a resume leg that keyed off the class names, matched zero sidecars, and passed green having resumed nothing; issue #4535 renamed them. The residual mapping is `PerFileRescoring` vs `PerFileRescore`, plus the `Fdr`/`FDR` casing that follows this codebase's type convention (`FdrEntry`, `FdrController`) rather than the all-caps `pwiz.Osprey.FDR` namespace. Folding those two in as well would let `ResolveTask` and `TaskCliName` be deleted outright. Evidence: the `HpcTask` enum in `Osprey.Core/OspreyConfig.cs`, `ResolveTask` / `TaskCliName` in `Osprey/Program.cs`. Severity: info. -- **[INTENTIONAL-CSHARP-DESIGN] Stage 6 writes a separate `.scores-reconciled.parquet`, not an in-place rewrite** - Rust doc says Stage 6 "rewrites each `.scores.parquet`" with reconciled scores; C# writes a separate `.scores-reconciled.parquet` sibling and leaves the Stage 4 parquet intact (crash-safety: a partial Stage 6 crash cannot half-rewrite the Stage 4 output). Each run's effective parquet then prefers the reconciled sibling (`ParquetScoreCache.EffectiveScoresPathFromScoresPath`). Evidence: `Osprey.Tasks/PerFileRescoreTask.cs`, `Osprey.Tasks/ScoringTaskShared.ScoresPathsForInputs`. Severity: minor. +- **[INTENTIONAL-CSHARP-DESIGN] Stage 6 writes a separate `.scores-reconciled.parquet`, not an in-place rewrite** - Rust doc says Stage 6 "rewrites each `.scores.parquet`" with reconciled scores; C# writes a separate `.scores-reconciled.parquet` sibling and leaves the Stage 4 parquet intact (crash-safety: a partial Stage 6 crash cannot half-rewrite the Stage 4 output). Which of the two a run is read from is decided by the task (`ScoringTaskShared.ReadsReconciledScores`). Evidence: `Osprey.Tasks/PerFileRescoreTask.cs`, `Osprey.Tasks/ScoringTaskShared.ScoresPathsForInputs`. Severity: minor. - **[INTENTIONAL-CSHARP-DESIGN] Orchestration is membership-predicate + lazy-rehydrate, not a stage window** - Rust doc frames each mode as "run stages X through Y, load the rest from disk"; C# implements a fixed four-task canonical pipeline where each task's `IsIncluded` decides participation and excluded/valid tasks lazy-rehydrate their state on demand through the typed byproduct registry. Behavior/outputs match the Rust modes (pinned by the membership truth table). Evidence: `Osprey/AnalysisPipeline.cs:99-148`, `Osprey.Test/PipelineMembershipTest.cs:55-93`. Severity: info. diff --git a/pwiz_tools/Osprey/docs/16-determinism.md b/pwiz_tools/Osprey/docs/16-determinism.md index 51f3ed59f1..27ac5b01d7 100644 --- a/pwiz_tools/Osprey/docs/16-determinism.md +++ b/pwiz_tools/Osprey/docs/16-determinism.md @@ -194,11 +194,12 @@ on it. The scoring task orders entries and writes them to the per-file `PerFileRescoreTask.SortFileEntriesCanonical` (`Osprey.Tasks/PerFileRescoreTask.cs:1301`) re-imposes the exact `(EntryId, Charge, ScanNumber, ParquetIndex)` order a cold run establishes, with `ParquetIndex` as a unique terminal key so the sort never -ties (`:1306-1315`). The comment at `:1287-1299` explains why this is applied to -**every** file (even no-work files with no reconciled Parquet): otherwise +ties (`:1306-1315`). Its comment explains why this is applied to **every** file the +resume overlays, including a file with no reconciliation work: otherwise `SecondPassFDR`'s `BuildSharedBoundaries` could iterate a different order and, on a q-value tie between charge states, pick a different shared `(modseq, file)` -boundary. Parquet preserves exact IEEE-754 values, so a rehydrated entry is +boundary. A file the resume loads from its reconciled Parquet instead arrives in +that order already, from `FirstPassSurvivorLoader`'s own canonical sort. Parquet preserves exact IEEE-754 values, so a rehydrated entry is bit-identical to the in-memory original (see 14-intermediate-files.md). The PEP estimator is fed a `base_id`-ascending-sorted union so its diff --git a/pwiz_tools/Osprey/regression.ps1 b/pwiz_tools/Osprey/regression.ps1 index 8114636f08..032e68eb3f 100644 --- a/pwiz_tools/Osprey/regression.ps1 +++ b/pwiz_tools/Osprey/regression.ps1 @@ -1701,10 +1701,11 @@ function Invoke-HpcChain { # worker dirs are done. foreach ($d in $ph3Dirs.Values) { Remove-Scratch $d } Copy-LibraryInto -Library $Library -Dir $ph4 -Manifest $Manifest - # -i again, and the RECONCILED parquet is what each run resolves to: this directory - # holds only the reconciled sibling, and EffectiveScoresPathFromScoresPath prefers it. - # Naming it explicitly is what --input-scores did; deriving it is what every other - # reader on this leg already did. + # -i again, and the RECONCILED parquet is what each run resolves to - because the TASK + # says so (ScoringTaskShared.ReadsReconciledScores), not because it is the only file + # here. That this directory also holds only the reconciled sibling is the enforcement + # above, and the two are deliberately independent: if the resolution ever regressed to + # a disk probe, the staging would hide it. $a4 = @('--task', 'SecondPassFDR') foreach ($s in $stemList) { $a4 += @('-i', "$s.mzML") } $a4 += @('-l', $libName, '-o', 'output.blib', '--resolution', $Resolution, @@ -1772,8 +1773,22 @@ foreach ($name in $selected) { # CanStreamStage7Join's first term, and only --task SecondPassFDR sets it, so the ordinary # run could not stream by construction. Deriving the admission from the reconciled parquets # on disk is what puts every leg under one question. + # + # OSPREY_STAGE6_STREAM_SURVIVORS=0 belongs here too, and its absence made this list + # wrong rather than merely incomplete. Under that switch BuildRunPerRunSource returns + # null (no loader, and the per-run lists are never cleared), so no marker reaches + # straight.log and the mode1 leg FAILS a run behaving exactly as instructed - while + # mode2/mode5 PASS, because BuildResumePerRunSource reads the loader through + # PublishedSurvivorLoader, which deliberately bypasses the Stage-6 switch. A mode1-only + # red that reads like a genuine regression is the worst shape a gate can have. + # + # STILL INCOMPLETE, deliberately, and worth knowing: this reads env vars only, while two + # of the three NeedsResidentPool triggers are CLI/config (--fdrbench-pass 1, a + # non-Percolator --fdr-method). A dataset spec setting either would red all three legs. + # No spec does today; if one is added, this has to grow a $cfg term. $cannotStreamJoin = ($env:OSPREY_STAGE7_STREAM -eq '0') -or + ($env:OSPREY_STAGE6_STREAM_SURVIVORS -eq '0') -or ($env:OSPREY_FDR_PROJECTION -eq '0') -or (-not [string]::IsNullOrWhiteSpace($env:OSPREY_PASS2_QVALUE) -and $env:OSPREY_PASS2_QVALUE -ne 'protein-compact') @@ -2711,7 +2726,24 @@ foreach ($name in $selected) { $legStream = Test-LogMarker -LogPath $legPath -Marker $stage7StreamMarker ` -Description ("$($streamLeg.What) folding Stage 7 one run at a time instead " + 'of rebuilding every run''s survivors at once') - if ($legStream.Pass) { + # PRESENCE OF THE MARKER IS NOT ENOUGH, and this is the hole it left. The marker is + # logged when the per-run source is BUILT, not when anything folds through it. A + # consumer that reads RescoredEntries.Value instead of streaming routes to + # MaterializeAllFromSource, which builds every run at once - the exact O(runs x + # entries) peak, 91.1 GB at 446 - while Streams stays true, so the marker is present, + # WarnResidentStage7Join stays silent, and this leg reported PASS on a resident run. + # MaterializeAllFromSource has always logged its own warning; nothing asserted its + # absence. Assert it here: the marker says a source was offered, this says nothing + # took the whole pool anyway. + $legPooled = Select-String -LiteralPath $legPath -SimpleMatch -Quiet ` + -Pattern 'a consumer asked for the whole-run survivor pool' + if ($legPooled) { + $overallFail = $true + Write-Problem-Tc ("$name $($streamLeg.Mode) (streamed join): FAIL - a per-run " + + 'source was published AND a consumer then pulled the whole pool through it, ' + + 'so the fold did not bound anything. The marker alone cannot see this.') + $summaryLines.Add("$name $($streamLeg.Mode) (streamed join): FAIL") + } elseif ($legStream.Pass) { $summaryLines.Add(("$name $($streamLeg.Mode) (streamed join): PASS " + '(per-run fold, no all-runs pool)')) } else { From d49b7556a2021f970b94723c067e88eee7df64c7 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 22:09:33 -0700 Subject: [PATCH 28/30] Moved BuildRescoredPool's summary back off the per-run source * The doc block described the deferred whole-run build and its [STAGE-WALL] line, but had been left attached to BuildRunPerRunSource, which carries its own summary and does neither See TODO-20260908_osprey_stage7_straightthrough_stream.md in pwiz-ai/todos Co-Authored-By: Claude --- .../Osprey/Osprey.Tasks/PerFileRescoreTask.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs index c8ab27780e..fbda99a74f 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/PerFileRescoreTask.cs @@ -2727,22 +2727,6 @@ public static RescoredPoolPlan RefillOnly( public IReadOnlyDictionary> ResetEntryIds { get; } } - /// - /// Bring the shared buffer to the whole-run post-rescore state the - /// milestone promises, on the first read of that - /// milestone (see the deferring constructor in PipelineByproducts.cs). - /// - /// This is the join work that used to end . It is here - /// because the pool is global - the pass-2 protein-compact competition is over a - /// global stratum - while is a per-file HPC task that exits when - /// its one file is done. Running it on the pull puts the cost on the consumer that - /// needs the pool, and lets a --task PerFileRescoring worker skip it by never - /// pulling (issue #4597). - /// - /// Reports its own [STAGE-WALL] line. The work left Stage 6's stopwatch - /// and lands inside no other stage's, so without one a perf comparison reads a 16-minute - /// Stage 6 saving with nothing anywhere absorbing it. - /// /// /// The per-run source Stage 7 folds through on the straight-through COMPUTE path, or /// null when this run cannot supply one and the whole-run @@ -2839,6 +2823,22 @@ private RescoredPoolPlan PoolPlanForBuild() return _poolPlan; } + /// + /// Bring the shared buffer to the whole-run post-rescore state the + /// milestone promises, on the first read of that + /// milestone (see the deferring constructor in PipelineByproducts.cs). + /// + /// This is the join work that used to end . It is here + /// because the pool is global - the pass-2 protein-compact competition is over a + /// global stratum - while is a per-file HPC task that exits when + /// its one file is done. Running it on the pull puts the cost on the consumer that + /// needs the pool, and lets a --task PerFileRescoring worker skip it by never + /// pulling (issue #4597). + /// + /// Reports its own [STAGE-WALL] line. The work left Stage 6's stopwatch + /// and lands inside no other stage's, so without one a perf comparison reads a 16-minute + /// Stage 6 saving with nothing anywhere absorbing it. + /// private void BuildRescoredPool(PipelineContext ctx) { var plan = PoolPlanForBuild(); From 07a56501655aabdcc431d7665c96e629b4adfdf2 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 23:13:53 -0700 Subject: [PATCH 29/30] Stopped the parallel gate counting its own warnings as failures * Select-String is case-insensitive by default, so ': FAIL' matched the ': fail' in a 'WARN: failed to prune' line and reported a lane that exited 0 with all 23 legs passing as 1 FAIL, failing the whole gate * Counted the leg lines case-sensitively and surfaced WARN lines separately See TODO-20260908_osprey_stage7_straightthrough_stream.md in pwiz-ai/todos Co-Authored-By: Claude --- pwiz_tools/Osprey/regression-parallel.ps1 | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pwiz_tools/Osprey/regression-parallel.ps1 b/pwiz_tools/Osprey/regression-parallel.ps1 index 76671b74f3..569cf697c2 100644 --- a/pwiz_tools/Osprey/regression-parallel.ps1 +++ b/pwiz_tools/Osprey/regression-parallel.ps1 @@ -158,17 +158,29 @@ Write-Host '=== Parallel regression summary ===' -ForegroundColor Cyan $totalPass = 0; $totalFail = 0; $totalSkip = 0; $worst = 0 foreach ($r in $running) { $text = if (Test-Path $r.Log) { Get-Content $r.Log } else { @() } - $pass = @($text | Select-String -Pattern ': PASS').Count - $fail = @($text | Select-String -Pattern ': FAIL').Count - $skip = @($text | Select-String -Pattern ': SKIP').Count + # -CaseSensitive, and it is not a nicety. Select-String is case-INSENSITIVE by default, + # so ': FAIL' matched the ': fail' inside any "WARN: failed to ..." line the lane emitted - + # and one of those (a prune racing a previous run's directory) turned a lane that exited 0, + # passed all 23 legs and printed "Osprey regression PASSED" into "1 FAIL" and an overall + # FAILED. A gate that cries wolf about its own warnings is worse than one that stays quiet: + # the next red gets read as this one. The leg lines these count are emitted in upper case by + # regression.ps1, so requiring that costs nothing. + $pass = @($text | Select-String -CaseSensitive -Pattern ': PASS').Count + $fail = @($text | Select-String -CaseSensitive -Pattern ': FAIL').Count + $skip = @($text | Select-String -CaseSensitive -Pattern ': SKIP').Count $totalPass += $pass; $totalFail += $fail; $totalSkip += $skip $code = $r.Proc.ExitCode if ($code -gt $worst) { $worst = $code } $colour = if ($code -eq 0 -and $fail -eq 0) { 'Green' } else { 'Red' } Write-Host (" {0,-45} exit={1} {2} PASS / {3} FAIL / {4} SKIP" -f $r.Name, $code, $pass, $fail, $skip) -ForegroundColor $colour - foreach ($line in ($text | Select-String -Pattern ': (PASS|FAIL|SKIP)')) { + foreach ($line in ($text | Select-String -CaseSensitive -Pattern ': (PASS|FAIL|SKIP)')) { Write-Host (" " + $line.Line.Trim()) } + # Warnings still surface - they were only ever miscounted, not unwanted - but as + # warnings, in their own colour, where nothing tallies them as legs. + foreach ($line in ($text | Select-String -CaseSensitive -Pattern '^\s*WARN:')) { + Write-Host (" " + $line.Line.Trim()) -ForegroundColor Yellow + } } Write-Host '' Write-Host (" TOTAL {0} PASS / {1} FAIL / {2} SKIP in {3:hh\:mm\:ss} wall" -f From 3cc4fa795696a0026501c38a312014c25adc7684 Mon Sep 17 00:00:00 2001 From: brendanx67 Date: Tue, 8 Sep 2026 23:21:23 -0700 Subject: [PATCH 30/30] Recorded why ResolveSidecarBasePath's fallback is not safe to delete * The unreachability argument conflicts with the SUPPORTED state where a run's file_name has no input_files stem, which is the case that branch answers * Led the flag-retirement rationale with the basenames framing: every per-run artifact is ., so --input-scores named the same set redundantly See TODO-20260908_osprey_input_scores_retirement.md in pwiz-ai/todos Co-Authored-By: Claude --- pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs | 12 ++++++++++-- pwiz_tools/Osprey/docs/15-hpc-scoring-split.md | 8 +++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs index 6a25d9e233..98caff453d 100644 --- a/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs +++ b/pwiz_tools/Osprey/Osprey.Tasks/ScoringTaskShared.cs @@ -287,8 +287,16 @@ internal static string ResolveSidecarBasePath( } } } - // Unreachable since --input-scores retired - see the remarks. Left in - // place rather than deleted mid-review-round. + // KEPT, and the "unreachable since --input-scores retired" note that stood here + // is not safe to act on. The argument for unreachability is that every task + // requires --input and the fileName keys are derived from those same inputs, so + // the loop above always matches. But PerFileRescoreTask documents a SUPPORTED + // state in which a run's file_name has no input_files stem (WriteUnchangedReconciled + // returns silently for it), and that is exactly the case this branch answers - + // deleting it would turn a synthesized sidecar path into null for the one shape + // that needs it. Establish which of the two is true before removing this; it is a + // behaviour change either way, and it belongs with the finding that owns that + // state rather than with the flag retirement that made it look dead. if (perFileParquetPaths != null && perFileParquetPaths.TryGetValue(fileName, out string parquetPath)) { diff --git a/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md b/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md index 1157b27a32..15540828b9 100644 --- a/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md +++ b/pwiz_tools/Osprey/docs/15-hpc-scoring-split.md @@ -125,7 +125,13 @@ Every task takes `-i` / `--input-list`, naming the **data files**, and derives e - **Order is the caller's.** FirstPassFDR reconciliation is order-sensitive, so a chain must pass a deterministically sorted list. `--input-scores` used to sort a globbed directory Ordinal on the caller's behalf; naming the runs explicitly means an orchestrator states the order rather than inheriting it from a directory listing, and a stray parquet in that directory can no longer change the cohort. - `--input-list` takes one path per line (blank lines and `#` comments ignored) and composes with `-i`. It is what a cohort past a few hundred runs needs: 446 `-i` paths measured ~28,600 characters against a 32,767 limit. -**Why the flag went.** It named an input KIND - "you handed me parquets" - which is how the Rust pipeline said *Stage 1-4 is done*. The C# port says that with `--task` plus the per-run validity sidecars, and two seams answering one question is what let `--task ModelDiagnostics` (a C#-era signal, while its inputs were mzML stems, a Rust-era one) join the pipeline and demand state a diagnostics fold never publishes. The clearest evidence it was a round trip: the pipeline's first act was `RescoreHydration.SyntheticInputFromParquet`, rebuilding a synthetic `.mzML` that does not exist, purely so the sidecar path helpers could work. +**Why the flag went.** The pipeline is a set of BASENAMES. Every per-run artifact is +`.` and every per-run map is keyed on the stem, so naming the runs names +everything they own. `--input-scores` named that same set with a different extension - it was +redundant by construction, not merely superseded, and the two spellings could disagree about +which runs the cohort contained. + +It also named an input KIND - "you handed me parquets" - which is how the Rust pipeline said *Stage 1-4 is done*. The C# port says that with `--task` plus the per-run validity sidecars, and two seams answering one question is what let `--task ModelDiagnostics` (a C#-era signal, while its inputs were mzML stems, a Rust-era one) join the pipeline and demand state a diagnostics fold never publishes. The clearest evidence it was a round trip: the pipeline's first act was `RescoreHydration.SyntheticInputFromParquet`, rebuilding a synthetic `.mzML` that does not exist, purely so the sidecar path helpers could work. ## Parquet footer hash validation