diff --git a/pwiz-sharp/Tools/Commandline/MsConvert/src/ArgParser.cs b/pwiz-sharp/Tools/Commandline/MsConvert/src/ArgParser.cs index 5c7fb4d814a..c4375e92350 100644 --- a/pwiz-sharp/Tools/Commandline/MsConvert/src/ArgParser.cs +++ b/pwiz-sharp/Tools/Commandline/MsConvert/src/ArgParser.cs @@ -70,6 +70,10 @@ internal static MsConvertConfig Parse(IReadOnlyList args) case "--mzMLb": config.WriteConfig.Format = WriteFormat.MzMLb; break; + case "--mzpeak": + case "--mzPeak": + config.WriteConfig.Format = WriteFormat.MzPeak; + break; case "--mgf": case "--MGF": config.WriteConfig.Format = WriteFormat.Mgf; @@ -433,7 +437,9 @@ private static string RequireNext(IReadOnlyList args, ref int i, string " --mzML mzML 1.1 (default)", " --mgf Mascot Generic Format", " --mzXML mzXML 3.2", - " --mz5, --mzMLb, --text (unimplemented)", + " --mzMLb mzMLb (HDF5)", + " --mzpeak mzPeak (Parquet archive)", + " --mz5, --text (unimplemented)", " --ms1, --bms1, --cms1, --ms2, --bms2, --cms2", "", "Filters:", diff --git a/pwiz-sharp/Tools/Commandline/MsConvert/src/Converter.cs b/pwiz-sharp/Tools/Commandline/MsConvert/src/Converter.cs index 62a03b8c197..1ab7aeaf53b 100644 --- a/pwiz-sharp/Tools/Commandline/MsConvert/src/Converter.cs +++ b/pwiz-sharp/Tools/Commandline/MsConvert/src/Converter.cs @@ -356,6 +356,7 @@ private string BuildOutputPath(string input, MSData msd) WriteFormat.MzXml => ".mzXML", WriteFormat.Mz5 => ".mz5", WriteFormat.MzMLb => ".mzMLb", + WriteFormat.MzPeak => ".mzpeak", WriteFormat.Mgf => ".mgf", WriteFormat.Text => ".txt", WriteFormat.Ms1 => ".ms1", diff --git a/pwiz-sharp/pwiz/src/MsData/DefaultReaderList.cs b/pwiz-sharp/pwiz/src/MsData/DefaultReaderList.cs index e1c1b564b7f..6246c97314f 100644 --- a/pwiz-sharp/pwiz/src/MsData/DefaultReaderList.cs +++ b/pwiz-sharp/pwiz/src/MsData/DefaultReaderList.cs @@ -126,6 +126,11 @@ public static ReaderList Default list.Add(new MSnReaderAdapter()); list.Add(new BtdxReaderAdapter()); list.Add(new MgfReaderAdapter()); + // Parquet-backed mzPeak goes at the end of the built-in chain so + // it doesn't perturb the existing identify-order for the XML/HDF5 + // formats. Its extension-first Identify makes the position + // immaterial for typed inputs. + list.Add(new MzPeakReaderAdapter()); foreach (var r in AdditionalReaders) list.Add(r); return list; diff --git a/pwiz-sharp/pwiz/src/MsData/Diff/MSDataDiff.cs b/pwiz-sharp/pwiz/src/MsData/Diff/MSDataDiff.cs index 6702640854b..0bc93424505 100644 --- a/pwiz-sharp/pwiz/src/MsData/Diff/MSDataDiff.cs +++ b/pwiz-sharp/pwiz/src/MsData/Diff/MSDataDiff.cs @@ -39,6 +39,51 @@ public static string Describe(MSData a, MSData b, DiffConfig? config = null) return ctx.Format(); } + /// + /// Full-metadata diff for a write→read round-trip through an mzML-complete binary format + /// (mzMLb, mzPeak). Compares the whole document at tolerance + /// (to absorb the format's float32 intensity narrowing) while tolerating the artifacts a + /// writer legitimately adds on output that aren't part of the source document: + /// + /// the output file added as a trailing sourceFile self-reference, + /// a conversion dataProcessing entry the writer stamps in, + /// mzMLb's per-array MS_external_* dataset/offset/length cvParams. + /// + /// Genuine metadata losses (entries present in but missing from + /// ) are still reported. + /// + public static string DescribeRoundTrip(MSData original, MSData roundtripped, double precision) + { + ArgumentNullException.ThrowIfNull(original); + ArgumentNullException.ThrowIfNull(roundtripped); + + // Drop writer-added sourceFiles / dataProcessings (present in the round-tripped doc but + // not the source — keyed by decoded id). Real losses surface as a-only and are untouched. + var origSourceFileIds = original.FileDescription.SourceFiles + .Select(s => DecodeXmlId(s.Id)).ToHashSet(StringComparer.Ordinal); + roundtripped.FileDescription.SourceFiles + .RemoveAll(s => !origSourceFileIds.Contains(DecodeXmlId(s.Id))); + + // De-duplicate dataProcessings by id first: MSDataFile.FillInCommonMetadata stamps a + // conversion entry on every read, so a round-tripped doc carries it twice (once embedded + // from the source, once re-added on read-back) under the same id. Then drop any remaining + // writer-added entries not present in the source. + DedupeById(roundtripped.DataProcessings, d => d.Id); + var origDataProcessingIds = original.DataProcessings + .Select(d => DecodeXmlId(d.Id)).ToHashSet(StringComparer.Ordinal); + roundtripped.DataProcessings + .RemoveAll(d => !origDataProcessingIds.Contains(DecodeXmlId(d.Id))); + + var dc = new DiffConfig { Precision = precision, IgnoreVersions = true }; + return Describe(original, roundtripped, dc); + } + + private static void DedupeById(List items, Func keyOf) + { + var seen = new HashSet(StringComparer.Ordinal); + items.RemoveAll(item => !seen.Add(DecodeXmlId(keyOf(item)))); + } + /// /// Tolerance mode for the msLevel comparison in . /// Captures the well-known lossy defaults each peak-list format applies on read. @@ -412,8 +457,8 @@ private static void DiffMobilityTriples(Spectrum a, Spectrum b, Context ctx) private static void DiffArrayMetadata(BinaryDataArray a, BinaryDataArray b, Context ctx) { - var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList(); - var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList(); + var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList(); + var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList(); DiffCvParamLists(aCvParams, bCvParams, ctx); DiffUserParamLists(a.UserParams, b.UserParams, ctx); } @@ -520,8 +565,8 @@ private static void DiffBinaryArray(BinaryDataArray a, BinaryDataArray b, Contex { // Filter out purely-serialization cvParams (32-bit / 64-bit precision, compression type) // before diffing — they describe how the array was encoded, not its content. - var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList(); - var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList(); + var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList(); + var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList(); DiffCvParamLists(aCvParams, bCvParams, ctx); // UserParams and ParamGroup refs still matter; compare them raw. DiffUserParamLists(a.UserParams, b.UserParams, ctx); @@ -541,6 +586,16 @@ private static void DiffBinaryArray(BinaryDataArray a, BinaryDataArray b, Contex } } + // mzMLb stores binary arrays out-of-line in HDF5 and points to them with these per-array + // cvParams (dataset name + offset + length). They're an internal mzMLb mechanism, not source + // metadata, so a round-trip diff against a non-mzMLb source must ignore them. + private static bool IsExternalBinaryRefCv(CVID cvid) + { + return cvid is CVID.MS_external_HDF5_dataset + or CVID.MS_external_offset + or CVID.MS_external_array_length; + } + private static bool IsBinaryEncodingCv(CVID cvid) { return cvid is CVID.MS_32_bit_float @@ -578,8 +633,8 @@ private static void DiffUserParamLists(List a, List b, Con private static void DiffIntegerArray(IntegerDataArray a, IntegerDataArray b, Context ctx) { // Same encoding-CV filtering as BinaryDataArray — int32/int64 choice is serialization. - var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList(); - var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList(); + var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList(); + var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList(); DiffCvParamLists(aCvParams, bCvParams, ctx); DiffUserParamLists(a.UserParams, b.UserParams, ctx); diff --git a/pwiz-sharp/pwiz/src/MsData/MSDataFile.cs b/pwiz-sharp/pwiz/src/MsData/MSDataFile.cs index 83052b4f394..300ce609ae9 100644 --- a/pwiz-sharp/pwiz/src/MsData/MSDataFile.cs +++ b/pwiz-sharp/pwiz/src/MsData/MSDataFile.cs @@ -168,6 +168,13 @@ public static void Write(MSData msd, string path, WriteConfig config, IterationL return; } + // mzPeak is also path-bound: a ZIP archive of Parquet tables built via random file I/O. + if (config.Format == WriteFormat.MzPeak) + { + Pwiz.Data.MsData.MzPeak.WriterMzPeak.Write(msd, path); + return; + } + using Stream output = OpenOutputStream(path, config.Gzip); Write(msd, output, config, ilr); } diff --git a/pwiz-sharp/pwiz/src/MsData/MsData.csproj b/pwiz-sharp/pwiz/src/MsData/MsData.csproj index cb4bf9ade34..c8ad3695d6e 100644 --- a/pwiz-sharp/pwiz/src/MsData/MsData.csproj +++ b/pwiz-sharp/pwiz/src/MsData/MsData.csproj @@ -26,6 +26,10 @@ to read/write HDF5-backed mass-spec files. The 1.10 stream tracks HDF5 1.10.x — the same major version pwiz cpp builds against. --> + + diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/ChromatogramList_MzPeak.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/ChromatogramList_MzPeak.cs new file mode 100644 index 00000000000..ea3ddfb1c3f --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/ChromatogramList_MzPeak.cs @@ -0,0 +1,191 @@ +using System; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.MsData.Processing; +using Pwiz.Data.MsData.Spectra; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// Lazy over an . +/// Sibling of . The mzPeak chromatogram model +/// is much simpler than the spectrum model — id, type CURIE, and the +/// (time, intensity) point arrays — so the translation here is short. +/// +internal sealed class ChromatogramList_MzPeak : ChromatogramListBase +{ + private readonly MzPeakReader _reader; + private readonly bool _ownsReader; + private readonly DataProcessing? _dp; + private readonly ChromatogramIdentity[] _identities; + + public ChromatogramList_MzPeak(MzPeakReader reader, DataProcessing? dp, bool ownsReader) + { + ArgumentNullException.ThrowIfNull(reader); + _reader = reader; + _ownsReader = ownsReader; + _dp = dp; + _identities = new ChromatogramIdentity[reader.ChromatogramCount]; + for (int i = 0; i < reader.ChromatogramCount; i++) + { + var desc = reader.GetChromatogramDescription(i); + _identities[i] = new ChromatogramIdentity { Index = i, Id = desc.Id }; + } + } + + public override int Count => _identities.Length; + + public override ChromatogramIdentity ChromatogramIdentity(int index) => _identities[index]; + + public override DataProcessing? DataProcessing => _dp; + + public override Chromatogram GetChromatogram(int index, bool getBinaryData = false) + { + if ((uint)index >= (uint)_identities.Length) + throw new ArgumentOutOfRangeException(nameof(index)); + + var desc = _reader.GetChromatogramDescription(index); + var chrom = new Chromatogram + { + Index = index, + Id = desc.Id, + }; + + // Chromatogram type CURIE → CV term. mzPeak stores e.g. MS:1000235 + // (total ion current chromatogram). Set both the bag-level type + // tag and any chromatogram-type-specific defaults the cpp readers emit. + var typeCvid = CvidFromCurie(desc.ChromatogramTypeCurie); + if (typeCvid != CVID.CVID_Unknown) + chrom.Params.Set(typeCvid); + + if (desc.Parameters is not null) + ApplyParams(chrom.Params, desc.Parameters); + + if (getBinaryData) + { + // pwiz/mzML chromatograms always carry a time + intensity array pair, even when empty; + // emit them unconditionally so 0-point chromatograms round-trip with the right shape. + var data = _reader.GetChromatogramData(index); + var time = data?.Time ?? Array.Empty(); + var intensitySrc = data?.Intensity ?? Array.Empty(); + var intensityDouble = new double[intensitySrc.Length]; + for (int i = 0; i < intensitySrc.Length; i++) intensityDouble[i] = intensitySrc[i]; + var timeUnit = CvidFromCurie(desc.TimeUnitCurie); + if (timeUnit == CVID.CVID_Unknown) timeUnit = CVID.UO_minute; + var intensityUnit = CvidFromCurie(desc.IntensityUnitCurie); + if (intensityUnit == CVID.CVID_Unknown) intensityUnit = CVID.MS_number_of_detector_counts; + SetTimeIntensityArrays(chrom, time, intensityDouble, timeUnit, intensityUnit); + MzPeakAuxArrays.Apply(desc.AuxArrays, chrom.BinaryDataArrays, chrom.IntegerDataArrays); + } + + return chrom; + } + + protected override void DisposeCore() + { + if (_ownsReader) _reader.Dispose(); + } + + /// + /// pwiz's doesn't expose a SetTimeIntensityArrays + /// helper (asymmetric with ), so + /// build the two BinaryDataArrays explicitly. Time uses MS_minute (matching + /// the cpp Chromatogram::set_time_intensity_arrays default). + /// + private static void SetTimeIntensityArrays(Chromatogram chrom, double[] time, double[] intensity, CVID timeUnit, CVID intensityUnit) + { + var timeArr = new BinaryDataArray(); + timeArr.Set(CVID.MS_time_array, "", timeUnit); + timeArr.Data.AddRange(time); + chrom.BinaryDataArrays.Add(timeArr); + + var intArr = new BinaryDataArray(); + intArr.Set(CVID.MS_intensity_array, "", intensityUnit); + intArr.Data.AddRange(intensity); + chrom.BinaryDataArrays.Add(intArr); + + chrom.DefaultArrayLength = time.Length; + } + + private static CVID CvidFromCurie(string? curie) + { + if (string.IsNullOrEmpty(curie)) return CVID.CVID_Unknown; + return CvLookup.CvTermInfo(curie).Cvid; + } + + /// Apply free-form chromatogram params (CV → CVParam, else UserParam, keeping type). + private static void ApplyParams(Pwiz.Data.Common.Params.ParamContainer target, System.Collections.Generic.IReadOnlyList src) + { + foreach (var p in src) + { + var cvid = CvidFromCurie(p.Accession); + var unitCvid = string.IsNullOrEmpty(p.Unit) ? CVID.CVID_Unknown : CvidFromCurie(p.Unit); + string value = p.ValueString + ?? p.ValueInteger?.ToString(System.Globalization.CultureInfo.InvariantCulture) + ?? p.ValueFloat?.ToString("R", System.Globalization.CultureInfo.InvariantCulture) + ?? (p.ValueBoolean is bool b ? (b ? "true" : "false") : string.Empty); + + if (cvid != CVID.CVID_Unknown) + target.Set(cvid, value, unitCvid); + else + target.UserParams.Add(new Pwiz.Data.Common.Params.UserParam( + p.Name ?? string.Empty, value, type: p.Type ?? string.Empty, units: unitCvid)); + } + } +} + +/// +/// Rebuilds auxiliary (non-canonical) binary/integer data arrays from their round-tripped +/// records onto a spectrum or chromatogram. Shared by +/// and . +/// +internal static class MzPeakAuxArrays +{ + public static void Apply( + System.Collections.Generic.IReadOnlyList? aux, + System.Collections.Generic.List binaryTarget, + System.Collections.Generic.List integerTarget) + { + if (aux is null) return; + foreach (var a in aux) + { + if (a.IsInteger) + { + var arr = new IntegerDataArray(); + ApplyMzPeakParams(arr, a.Params); + if (a.IntValues is not null) arr.Data.AddRange(a.IntValues); + integerTarget.Add(arr); + } + else + { + var arr = new BinaryDataArray(); + ApplyMzPeakParams(arr, a.Params); + if (a.DoubleValues is not null) arr.Data.AddRange(a.DoubleValues); + binaryTarget.Add(arr); + } + } + } + + internal static void ApplyMzPeakParams(Pwiz.Data.Common.Params.ParamContainer target, + System.Collections.Generic.IReadOnlyList src) + { + foreach (var p in src) + { + var cvid = string.IsNullOrEmpty(p.Accession) ? CVID.CVID_Unknown : CvLookup.CvTermInfo(p.Accession!).Cvid; + var unit = string.IsNullOrEmpty(p.Unit) ? CVID.CVID_Unknown : CvLookup.CvTermInfo(p.Unit!).Cvid; + string value = p.Value switch + { + null => string.Empty, + string s => s, + bool b => b ? "true" : "false", + long l => l.ToString(System.Globalization.CultureInfo.InvariantCulture), + int i => i.ToString(System.Globalization.CultureInfo.InvariantCulture), + double d => d.ToString("R", System.Globalization.CultureInfo.InvariantCulture), + _ => p.Value.ToString() ?? string.Empty, + }; + if (cvid != CVID.CVID_Unknown) + target.Set(cvid, value, unit); + else + target.UserParams.Add(new Pwiz.Data.Common.Params.UserParam(p.Name ?? string.Empty, value, type: string.Empty, units: unit)); + } + } +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/FileMetadata.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/FileMetadata.cs new file mode 100644 index 00000000000..38e96577f81 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/FileMetadata.cs @@ -0,0 +1,272 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// File-level metadata records — deserialized from the Parquet KV JSON strings in +/// spectra_metadata.parquet / chromatograms_metadata.parquet. The mzPeak writer +/// emits these once per file (not per row) because they describe the whole run: +/// instrument config, software stack, sample info, source files, etc. +/// +public sealed record FileMetadata( + FileDescription FileDescription, + IReadOnlyList InstrumentConfigurations, + IReadOnlyList DataProcessingMethods, + IReadOnlyList Software, + IReadOnlyList Samples, + RunInfo Run, + long SpectrumCount, + long SpectrumDataPointCount, + // Document-level mzML id (MSData.Id). Round-trips the attribute; cross-stack + // readers that don't model it simply ignore the extra KV entry. + string? DocumentId = null, + // referenceableParamGroupList: shared CV-param bundles spectra/scans reference by id. + IReadOnlyList? ParamGroups = null); + +public sealed record FileDescription( + IReadOnlyList Contents, + IReadOnlyList SourceFiles); + +public sealed record SourceFile( + string Id, + string Name, + string? Location, + IReadOnlyList Parameters); + +public sealed record InstrumentConfiguration( + int Id, + IReadOnlyList Components, + string? SoftwareReference, + IReadOnlyList Parameters, + // pwiz's instrument-configuration id is an arbitrary string (e.g. "LCQ Deca"); the columnar + // Id above is the cross-stack integer index. OriginalId preserves the string so the + // round-trip restores the real id and every scan/run reference that points at it. + string? OriginalId = null, + // referenceableParamGroup ids this instrument configuration references. + IReadOnlyList? ParamGroupRefs = null); + +/// One source / analyzer / detector in an instrument's component chain. +public sealed record ComponentInfo( + string Type, // "source" | "analyzer" | "detector" + int Order, + IReadOnlyList Parameters); + +public sealed record DataProcessingMethod( + string Id, + IReadOnlyList Methods); + +/// One ordered processing step within a . +public sealed record ProcessingMethodInfo( + int Order, + string? SoftwareReference, + IReadOnlyList Parameters); + +/// A referenceableParamGroup: an id plus the CV/user params it bundles. +public sealed record ParamGroupInfo( + string Id, + IReadOnlyList Parameters); + +/// +/// One auxiliary binary/integer data array on a spectrum or chromatogram — i.e. an array beyond +/// the canonical m/z+intensity (spectrum) or time+intensity (chromatogram) pair: ion-mobility +/// arrays, "ms level" non-standard arrays, resolution/baseline/SN arrays, etc. +/// carries the identifying CV/user params (array-type term, name, unit) minus the binary-encoding +/// terms. Exactly one of / is populated per +/// . Serialized as a JSON list in a per-row auxiliary_arrays column. +/// +public sealed record AuxiliaryArrayData( + IReadOnlyList Params, + bool IsInteger, + IReadOnlyList? DoubleValues, + IReadOnlyList? IntValues); + +/// JSON (de)serialization for the per-row auxiliary_arrays column. +public static class AuxiliaryArrays +{ + private static readonly JsonSerializerOptions Opts = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + /// Serialize a list of auxiliary arrays to a JSON string, or null when empty. + public static string? Serialize(IReadOnlyList? arrays) => + arrays is null || arrays.Count == 0 ? null : JsonSerializer.Serialize(arrays, Opts); + + /// Parse the JSON string back into auxiliary arrays; null/empty → empty list. + public static IReadOnlyList Parse(string? json) => + string.IsNullOrEmpty(json) + ? System.Array.Empty() + : JsonSerializer.Deserialize>(json, Opts) ?? new List(); +} + +/// +/// One extra scan (beyond scan[0]) of a spectrum's scanList — used for combined ion-mobility +/// spectra whose scanList holds one scan per mobility bin. Carries the scan's full CV/user params +/// and its scan windows (each a param list). scan[0] still rides the typed columns. +/// +public sealed record ExtraScanData( + IReadOnlyList Params, + IReadOnlyList> ScanWindows, + // The scan's spectrumRef (source spectrum for a combined scan), if any. + string? SpectrumId = null); + +/// JSON (de)serialization for the per-spectrum extra-scans column. +public static class ExtraScans +{ + private static readonly JsonSerializerOptions Opts = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + /// Serialize extra scans to JSON, or null when there are none. + public static string? Serialize(IReadOnlyList? scans) => + scans is null || scans.Count == 0 ? null : JsonSerializer.Serialize(scans, Opts); + + /// Parse extra scans; null/empty → empty list. + public static IReadOnlyList Parse(string? json) => + string.IsNullOrEmpty(json) + ? System.Array.Empty() + : JsonSerializer.Deserialize>(json, Opts) ?? new List(); +} + +/// +/// JSON (de)serialization for per-scan scan-window free-form params: a list (per scan window, in +/// order) of the window's CV/user params beyond the typed lower/upper limit columns. +/// +public static class ScanWindowParams +{ + private static readonly JsonSerializerOptions Opts = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + /// Serialize per-window param lists; null when every window's list is empty. + public static string? Serialize(IReadOnlyList>? windows) => + windows is null || windows.Count == 0 || windows.All(w => w.Count == 0) + ? null + : JsonSerializer.Serialize(windows, Opts); + + /// Parse back to per-window param lists; null/empty → empty list. + public static IReadOnlyList> Parse(string? json) + { + if (string.IsNullOrEmpty(json)) return System.Array.Empty>(); + // IReadOnlyList is covariant, so List> satisfies IReadOnlyList>. + return JsonSerializer.Deserialize>>(json, Opts) + ?? (IReadOnlyList>)System.Array.Empty>(); + } +} + +public sealed record SoftwareInfo( + string Id, + string? Version, + IReadOnlyList Parameters); + +public sealed record SampleInfo( + string Id, + string? Name, + IReadOnlyList Parameters); + +public sealed record RunInfo( + string Id, + string? DefaultDataProcessingId, + int? DefaultInstrumentId, // null when the run has no default instrument configuration + string? DefaultSourceFileId, + string? StartTime, + IReadOnlyList Parameters); + +/// +/// CvParam shape used in the file-level JSON blocks. Distinct from the columnar +/// Phase 2.5 because the JSON form has a single +/// polymorphic value field, whereas the Parquet columnar form splits it +/// into four typed columns (value.string / value.integer / value.float / +/// value.boolean). Same logical semantics, different on-disk encoding. +/// +public sealed record MzPeakCvParam( + string? Name, + string? Accession, + [property: JsonConverter(typeof(PolymorphicValueConverter))] object? Value, + string? Unit); + +/// +/// Maps the JSON value union (string | integer | floating | boolean | null) to +/// a concrete .NET type. We don't preserve a wrapping JsonElement — the caller +/// usually just wants the value, not the JSON kind. +/// +internal sealed class PolymorphicValueConverter : JsonConverter +{ + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => reader.TokenType switch + { + JsonTokenType.Null => null, + JsonTokenType.String => reader.GetString(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Number => reader.TryGetInt64(out var i) ? (object)i : reader.GetDouble(), + _ => throw new JsonException($"Unexpected token {reader.TokenType} for CvParam value"), + }; + + public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerOptions options) + { + switch (value) + { + case null: writer.WriteNullValue(); break; + case string s: writer.WriteStringValue(s); break; + case bool b: writer.WriteBooleanValue(b); break; + case long l: writer.WriteNumberValue(l); break; + case int i: writer.WriteNumberValue(i); break; + case double d: writer.WriteNumberValue(d); break; + case float f: writer.WriteNumberValue(f); break; + default: throw new JsonException($"Unsupported CvParam value type {value.GetType()}"); + } + } +} + +internal static class FileMetadataDeserializer +{ + private static readonly JsonSerializerOptions JsonOpts = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = true, + }; + + /// + /// Parse the KV strings from a parquet file's KeyValueMetadata into one + /// typed FileMetadata record. Throws on malformed input — these blocks + /// are written by the same writer that wrote the schema, so failures + /// here are programmer / corruption errors, not user input. + /// + public static FileMetadata Parse(IReadOnlyDictionary kv) + { + return new FileMetadata( + FileDescription: Get(kv, "file_description"), + InstrumentConfigurations: Get>(kv, "instrument_configuration_list"), + DataProcessingMethods: Get>(kv, "data_processing_method_list"), + Software: Get>(kv, "software_list"), + Samples: Get>(kv, "sample_list"), + Run: Get(kv, "run"), + SpectrumCount: GetLong(kv, "spectrum_count"), + SpectrumDataPointCount: GetLong(kv, "spectrum_data_point_count"), + DocumentId: GetStringOrNull(kv, "document_id"), + ParamGroups: GetOrNull>(kv, "referenceable_param_group_list")); + } + + private static T Get(IReadOnlyDictionary kv, string key) => + JsonSerializer.Deserialize(kv[key], JsonOpts) + ?? throw new InvalidDataException($"Required KV '{key}' deserialized to null."); + + private static T? GetOrNull(IReadOnlyDictionary kv, string key) where T : class => + kv.TryGetValue(key, out var json) ? JsonSerializer.Deserialize(json, JsonOpts) : null; + + private static string? GetStringOrNull(IReadOnlyDictionary kv, string key) => + kv.TryGetValue(key, out var v) && !string.IsNullOrEmpty(v) ? v : null; + + private static long GetLong(IReadOnlyDictionary kv, string key) => + long.Parse(kv[key], System.Globalization.CultureInfo.InvariantCulture); +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakArchive.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakArchive.cs new file mode 100644 index 00000000000..2b54fd8dd07 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakArchive.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.IO; +using ParquetSharp; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// Opens the parquet entries of a .mzpeak ZIP in place — without extracting the archive +/// to a temp directory — when every entry is Stored (uncompressed), which is how +/// writes them. Each entry's data is then a contiguous, seekable byte +/// range, so a can read it through a length-bounded sub-stream over +/// the archive file. This avoids copying the (often hundreds-of-MB) data parquet to disk just to +/// read a few row groups. Falls back to null (caller extracts) if any entry isn't Stored or +/// the central directory can't be parsed. +/// +internal sealed class MzPeakArchive : IDisposable +{ + private readonly string _path; + private readonly Dictionary _entries; + + private MzPeakArchive(string path, Dictionary entries) + { + _path = path; + _entries = entries; + } + + public bool HasEntry(string name) => _entries.ContainsKey(name); + + /// Open a parquet entry as a ParquetFileReader over a seekable sub-stream; null if absent. + public ParquetFileReader? OpenParquet(string name) + { + if (!_entries.TryGetValue(name, out var e)) return null; + var fs = new FileStream(_path, FileMode.Open, FileAccess.Read, FileShare.Read); + var sub = new BoundedSubStream(fs, e.Offset, e.Length); // disposes fs when disposed + return new ParquetFileReader(sub, leaveOpen: false); // disposes sub on Close + } + + public void Dispose() { /* OpenParquet streams are owned by their ParquetFileReader */ } + + /// + /// Parse the ZIP central directory and return Stored entries' (data offset, length). Returns + /// null if the archive isn't all-Stored or can't be parsed (caller falls back to extraction). + /// + public static MzPeakArchive? TryOpen(string path) + { + try + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + long fileLen = fs.Length; + int maxScan = (int)Math.Min(fileLen, 65557); // 64KB comment + 22-byte EOCD + var tail = new byte[maxScan]; + fs.Seek(fileLen - maxScan, SeekOrigin.Begin); + fs.ReadExactly(tail, 0, maxScan); + + int eocd = -1; + for (int i = maxScan - 22; i >= 0; i--) + if (tail[i] == 0x50 && tail[i + 1] == 0x4b && tail[i + 2] == 0x05 && tail[i + 3] == 0x06) { eocd = i; break; } + if (eocd < 0) return null; + + long cdOffset = U32(tail, eocd + 16); + long cdEntries = U16(tail, eocd + 10); + + // Zip64: when the 32-bit fields are saturated, the real values live in the Zip64 EOCD. + if (cdOffset == 0xFFFFFFFF || cdEntries == 0xFFFF) + { + int loc = -1; + for (int i = eocd - 20; i >= 0; i--) + if (tail[i] == 0x50 && tail[i + 1] == 0x4b && tail[i + 2] == 0x06 && tail[i + 3] == 0x07) { loc = i; break; } + if (loc < 0) return null; + long z64Eocd = (long)U64(tail, loc + 8); + var z64 = new byte[56]; + fs.Seek(z64Eocd, SeekOrigin.Begin); + fs.ReadExactly(z64, 0, 56); + if (!(z64[0] == 0x50 && z64[1] == 0x4b && z64[2] == 0x06 && z64[3] == 0x06)) return null; + cdEntries = (long)U64(z64, 32); + cdOffset = (long)U64(z64, 48); + } + + fs.Seek(cdOffset, SeekOrigin.Begin); + var entries = new Dictionary(StringComparer.Ordinal); + for (long e = 0; e < cdEntries; e++) + { + var hdr = new byte[46]; + fs.ReadExactly(hdr, 0, 46); + if (!(hdr[0] == 0x50 && hdr[1] == 0x4b && hdr[2] == 0x01 && hdr[3] == 0x02)) return null; + int method = U16(hdr, 10); + long uncompSize = U32(hdr, 24); + int nameLen = U16(hdr, 28); + int extraLen = U16(hdr, 30); + int commentLen = U16(hdr, 32); + long localOffset = U32(hdr, 42); + + var nameBytes = new byte[nameLen]; + fs.ReadExactly(nameBytes, 0, nameLen); + var extra = new byte[extraLen]; + fs.ReadExactly(extra, 0, extraLen); + if (commentLen > 0) fs.Seek(commentLen, SeekOrigin.Current); + + if (method != 0) return null; // not Stored → can't sub-stream; fall back to extraction + + // Zip64 extra field (id 0x0001) supplies any saturated size/offset, in field order. + if (uncompSize == 0xFFFFFFFF || localOffset == 0xFFFFFFFF) + { + bool ok = ReadZip64Extra(extra, U32(hdr, 20) == 0xFFFFFFFF, uncompSize == 0xFFFFFFFF, + localOffset == 0xFFFFFFFF, out long u64Uncomp, out long u64Offset); + if (!ok) return null; + if (uncompSize == 0xFFFFFFFF) uncompSize = u64Uncomp; + if (localOffset == 0xFFFFFFFF) localOffset = u64Offset; + } + + // Local header: data starts after its own (possibly different) name+extra lengths. + long savedCdPos = fs.Position; + var lh = new byte[30]; + fs.Seek(localOffset, SeekOrigin.Begin); + fs.ReadExactly(lh, 0, 30); + if (!(lh[0] == 0x50 && lh[1] == 0x4b && lh[2] == 0x03 && lh[3] == 0x04)) return null; + int lNameLen = U16(lh, 26); + int lExtraLen = U16(lh, 28); + long dataOffset = localOffset + 30 + lNameLen + lExtraLen; + fs.Seek(savedCdPos, SeekOrigin.Begin); + + string name = System.Text.Encoding.UTF8.GetString(nameBytes); + entries[name] = (dataOffset, uncompSize); + } + return entries.Count == 0 ? null : new MzPeakArchive(path, entries); + } + catch + { + return null; + } + } + + private static bool ReadZip64Extra(byte[] extra, bool needComp, bool needUncomp, bool needOffset, + out long uncomp, out long offset) + { + uncomp = 0; offset = 0; + int p = 0; + while (p + 4 <= extra.Length) + { + int id = U16(extra, p); + int size = U16(extra, p + 2); + int body = p + 4; + if (id == 0x0001) + { + int q = body; + if (needUncomp) { if (q + 8 > extra.Length) return false; uncomp = (long)U64(extra, q); q += 8; } + if (needComp) { q += 8; } // compressed size precedes offset + if (needOffset) { if (q + 8 > extra.Length) return false; offset = (long)U64(extra, q); } + return true; + } + p = body + size; + } + return false; + } + + private static int U16(byte[] b, int o) => b[o] | (b[o + 1] << 8); + private static long U32(byte[] b, int o) => (uint)(b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)); + private static ulong U64(byte[] b, int o) + { + ulong v = 0; + for (int i = 7; i >= 0; i--) v = (v << 8) | b[o + i]; + return v; + } + + /// Read-only seekable window [offset, offset+length) over an owned FileStream. + private sealed class BoundedSubStream : Stream + { + private readonly FileStream _inner; + private readonly long _offset; + private readonly long _length; + private long _pos; + + public BoundedSubStream(FileStream inner, long offset, long length) + { + _inner = inner; + _offset = offset; + _length = length; + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => _length; + + public override long Position + { + get => _pos; + set => _pos = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_pos >= _length) return 0; + int toRead = (int)Math.Min(count, _length - _pos); + _inner.Seek(_offset + _pos, SeekOrigin.Begin); + int n = _inner.Read(buffer, offset, toRead); + _pos += n; + return n; + } + + public override long Seek(long offset, SeekOrigin origin) + { + _pos = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _pos + offset, + SeekOrigin.End => _length + offset, + _ => _pos, + }; + return _pos; + } + + public override void Flush() { } + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) _inner.Dispose(); + base.Dispose(disposing); + } + } +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakChunkCodec.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakChunkCodec.cs new file mode 100644 index 00000000000..4eaf53e0f81 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakChunkCodec.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// Decoders for the mzPeak "chunked" point-buffer layout (mzPeak.NET's compressed variant), where +/// each parquet row holds one m/z chunk — a start value, an encoded list of values, an encoding +/// CURIE, and the parallel intensity list — instead of one row per point. +/// +/// This is a faithful port of mzPeak.NET's DeltaCodec/NoCompressionCodec + +/// NullInterpolation.FillNullsWithModel (MZPeakNet/Compute.cs). The m/z axis is delta-encoded +/// (cumulative sum from the chunk start) and may carry nulls at chunk seams; those nulls are filled +/// from a per-spectrum spacing-interpolation model (a polynomial whose coefficients are stored in the +/// spectrum's mz_delta_model metadata column) or, for wider gaps, the local median spacing. +/// Intensities are stored verbatim with nulls meaning zero. +/// +internal static class MzPeakChunkCodec +{ + public const string DeltaCurie = "MS:1003089"; // delta encoding + public const string NoCompressionCurie = "MS:1000576"; // no compression + public const string NullInterpolateCurie = "MS:1003901"; // nulls filled from the spacing model + public const string NullZeroCurie = "MS:1003902"; // nulls mean zero + + /// Decode one chunk's m/z list to absolute values (which may still contain nulls at the + /// seam where a chunk restarts from an absolute value). is the chunk's + /// chunk_encoding CURIE. + public static double?[] DecodeMz(string? encoding, double start, double?[] values) + { + return encoding switch + { + DeltaCurie => DecodeDelta(start, values), + NoCompressionCurie or null => DecodeNoCompression(start, values), + _ => throw new NotSupportedException( + $"mzPeak chunk encoding '{encoding}' is not supported (only delta {DeltaCurie} and " + + $"no-compression {NoCompressionCurie}; numpress-encoded chunks are not yet implemented)."), + }; + } + + private static double?[] DecodeNoCompression(double start, double?[] values) + { + var outv = new List(values.Length + 1) { start }; + outv.AddRange(values); + return outv.ToArray(); + } + + private static double?[] DecodeDelta(double start, double?[] values) + { + var outv = new List(values.Length + 1); + double? last = start; + if (values.Length > 0 && values[0] is null) + { + // The chunk restarts from an absolute value carried in values[1]; the leading slot is a + // null seam to be interpolated later. (When values[1] is also null the start is real.) + if (values.Length > 1 && values[1] is null) outv.Add(start); + last = null; + } + else + { + outv.Add(start); + } + + foreach (var v in values) + { + if (v is double d) + { + if (last is null) { last = d; outv.Add(d); } + else { last += d; outv.Add(last); } + } + else + { + last = null; + outv.Add(null); + } + } + return outv.ToArray(); + } + + /// Replace the null seams in a decoded m/z segment using the spacing model (polynomial + /// ) for narrow gaps and the local median spacing for wider ones. + public static double[] FillNullsWithModel(double?[] values, double[] coef) + { + var outList = new List(values.Length); + foreach (var (startIdx, endIdx) in FindNullBounds(values)) + { + // mzPeak.NET slices [startIdx, endIdx] inclusive; the trailing +1 is clamped to the array. + int len = Math.Min(endIdx - startIdx + 1, values.Length - startIdx); + var chunk = new double?[len]; + Array.Copy(values, startIdx, chunk, 0, len); + + int n = chunk.Length; + int nHasReal = n - chunk.Count(x => x is null); + + if (nHasReal == 1) + { + if (n == 2) + { + if (chunk[0] is null) + { + double vAt = chunk[1]!.Value; + outList.Add(vAt - Predict(coef, vAt)); + outList.Add(vAt); + } + else + { + double vAt = chunk[0]!.Value; + outList.Add(vAt); + outList.Add(vAt + Predict(coef, vAt)); + } + } + else if (n == 3) + { + double vAt = chunk[1]!.Value; + outList.Add(vAt - Predict(coef, vAt)); + outList.Add(vAt); + outList.Add(vAt + Predict(coef, vAt)); + } + else throw new InvalidOperationException("unreachable null-bound shape"); + } + else + { + double delta = LocalMedianDelta(chunk); + if (chunk[0] is null) outList.Add(chunk[1]!.Value - delta); + else outList.Add(chunk[0]!.Value); + + for (int j = 1; j <= chunk.Length - 2; j++) + { + if (chunk[j] is not double mid) throw new InvalidOperationException("interior null in chunk"); + outList.Add(mid); + } + + if (chunk[^1] is null) outList.Add(chunk[^2]!.Value + delta); + else outList.Add(chunk[^1]!.Value); + } + } + return outList.ToArray(); + } + + /// Decoded m/z with no nulls (or after a fill) → plain double[]; any residual null → NaN. + public static double[] ToDense(double?[] values) + { + var outv = new double[values.Length]; + for (int i = 0; i < values.Length; i++) outv[i] = values[i] ?? double.NaN; + return outv; + } + + /// Intensity list → dense float[]; nulls mean zero (the NullZero transform). + public static float[] IntensityToDense(float?[] values) + { + var outv = new float[values.Length]; + for (int i = 0; i < values.Length; i++) outv[i] = values[i] ?? 0f; + return outv; + } + + // Polynomial spacing model: coef[0] + coef[1]*x + coef[2]*x^2 + ... + private static double Predict(double[] coef, double value) + { + double acc = coef[0]; + for (int i = 1; i < coef.Length; i++) + { + double x = value; + for (int j = 1; j < i; j++) x *= value; + acc += x * coef[i]; + } + return acc; + } + + private static double LocalMedianDelta(double?[] chunk) + { + var deltas = CollectDeltas(chunk); + if (deltas.Count == 0) return 0.0; + double median = SortedMedian(deltas); + var below = deltas.Where(v => v <= median).ToList(); + return below.Count == 0 ? median : SortedMedian(below); + } + + private static List CollectDeltas(double?[] values) + { + var deltas = new List(); + double last = 0.0; + bool seen = false; + foreach (var value in values) + { + if (value is not double v) continue; + if (!seen) { last = v; seen = true; } + else + { + double delta = v - last; + if (delta < 0) throw new InvalidOperationException($"negative delta {delta} = {v} - {last}"); + deltas.Add(delta); + last = v; + } + } + deltas.Sort(); + return deltas; + } + + private static double SortedMedian(IReadOnlyList values) + { + if (values.Count == 0) return 0.0; + if (values.Count <= 2) return values[0]; + int mid = values.Count / 2; + return values.Count % 2 == 0 ? values[mid] : (values[mid] + values[mid + 1]) / 2.0; + } + + private static List<(int, int)> FindNullBounds(double?[] arr) + { + var bounds = new List<(int, int)>(); + if (arr.Length == 0) return bounds; + + var nullHere = new List(); + for (int i = 0; i < arr.Length; i++) + if (arr[i] is null) nullHere.Add(i); + + if (nullHere.Count == 0) { bounds.Add((0, arr.Length)); return bounds; } + + if (nullHere[0] != 0) nullHere.Insert(0, 0); + if (nullHere[^1] != arr.Length - 1) nullHere.Add(arr.Length); + if (nullHere.Count % 2 != 0) + throw new InvalidOperationException($"unpaired nulls in chunk ({nullHere.Count})"); + + for (int i = 0; i < nullHere.Count; i += 2) + bounds.Add((nullHere[i], nullHere[i + 1])); + return bounds; + } +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakReader.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakReader.cs new file mode 100644 index 00000000000..d4d7dfb3bf9 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakReader.cs @@ -0,0 +1,1799 @@ +using System.IO.Compression; +using System.Text.RegularExpressions; +using ParquetSharp; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// Phase 2 / 2.5 / 3 / 4 mzPeak reader prototype — synchronous, no async runtime touched. +/// +/// 2: scalar spectrum metadata (id, time, ms_level, profile/centroid CURIE, scan_start_time, filter_string). +/// 2.5: nested CvParam lists (spectrum.parameters, scan.parameters), isolation_window / +/// activation / selected_ion scalars. +/// 3: per-spectrum random-access binary data — mz + intensity arrays — from +/// spectra_data.parquet (profile) or spectra_peaks.parquet (centroid). +/// 4a: file-level metadata via Parquet KV JSON (file_description, instrument +/// configurations, software, samples, run info). +/// 4b: chromatograms (count, description, time/intensity arrays) — same column +/// dispatcher pattern as spectra, separate parquet files. +/// +/// Deferred to later phases: aux_arrays (rep=2), nested parameters under +/// isolation_window / activation / scan_windows (rep=2), multi-precursor spectra, +/// row-group-lazy reads (current code eager-loads the whole metadata table + +/// point-layout files), writer. +/// +public sealed class MzPeakReader : IDisposable +{ + public sealed record SpectrumDescription( + ulong Index, + string Id, + double Time, + int? MsLevel, + bool IsProfile, + bool IsCentroid, + string? RepresentationCurie, + ScanInfo? Scan, + IReadOnlyList Precursors, + IReadOnlyList Parameters, + // Phase 7 — MSData parity additions + int? ScanPolarity = null, + string? SpectrumTypeCurie = null, + long? NumberOfDataPoints = null, + long? NumberOfPeaks = null, + double? BasePeakMz = null, + double? BasePeakIntensity = null, + double? TotalIonCurrent = null, + double? LowestObservedMz = null, + double? HighestObservedMz = null, + string? DataProcessingRef = null, + int? NumberOfAuxiliaryArrays = null, + IReadOnlyList? MzDeltaModel = null, + // scanList combination method CURIE + referenceableParamGroup ids referenced by the spectrum. + string? ScanCombinationCurie = null, + IReadOnlyList? ParamGroupRefs = null, + IReadOnlyList? AuxArrays = null, + // Value-array type/unit CURIEs when the value array isn't m/z (UV/DAD wavelength, …). + string? ValueArrayCurie = null, + string? ValueArrayUnitCurie = null); + + public sealed record ScanInfo( + double? StartTime, + string? FilterString, + uint? InstrumentConfigurationRef, + double? IonInjectionTime, + IReadOnlyList Parameters, + IReadOnlyList ScanWindows, + // Phase 7 + ulong? SourceIndex = null, + double? IonMobilityValue = null, + string? IonMobilityTypeCurie = null, + long? PresetScanConfiguration = null, + // JSON of per-scan-window free-form params (parallel to ScanWindows). + string? ScanWindowParamsJson = null, + // JSON of scanList scans beyond this one (combined ion-mobility spectra). + string? ExtraScansJson = null, + // scan[0] spectrumRef (source spectrum for a combined scan), if any. + string? SpectrumRef = null); + + public sealed record PrecursorInfo( + string? PrecursorId, + IsolationWindow? IsolationWindow, + Activation? Activation, + SelectedIonInfo? SelectedIon, + // Phase 7 + ulong? SourceIndex = null, + ulong? PrecursorIndex = null); + + public sealed record IsolationWindow( + double? TargetMz, + double? LowerOffset, + double? UpperOffset, + IReadOnlyList Parameters); + + public sealed record Activation( + double? CollisionEnergy, + string? DissociationMethod, + IReadOnlyList Parameters); + + public sealed record SelectedIonInfo( + double? Mz, + double? PeakIntensity, + long? ChargeState, + IReadOnlyList Parameters, + // Phase 7 + ulong? SourceIndex = null, + ulong? PrecursorIndex = null, + double? IonMobilityValue = null, + string? IonMobilityTypeCurie = null); + + /// + /// One scan window — the m/z range the instrument was set to scan over for this + /// scan. cpp/mzML allows multiple scan windows per scan; the demo has exactly one + /// per scan. The nested rep=2 parameters inside each scan window are deferred + /// (would require LogicalReader<T?[][]>); the scalar lower/upper limits are + /// what matters for most consumers. + /// + public sealed record ScanWindow(double? LowerLimit, double? UpperLimit); + + /// + /// A controlled-vocabulary parameter. Exactly one of String/Integer/Float/Boolean + /// is non-null per the spec's value-union encoding; the C# side surfaces them + /// through the discriminated accessor. + /// + public sealed record CvParam( + string? Name, + string? Accession, + string? ValueString, + long? ValueInteger, + double? ValueFloat, + bool? ValueBoolean, + string? Unit, + // xsd datatype hint for userParams (e.g. "xsd:float"); null/empty for CV params. + string? Type = null) + { + public object? Value => + (object?)ValueString ?? (object?)ValueInteger ?? (object?)ValueFloat ?? (object?)ValueBoolean; + } + + /// + /// Mz + intensity arrays for one spectrum. The mzPeak writer puts every spectrum's + /// canonical data — profile signal for profile spectra, peak data for centroid + /// spectra — in spectra_data.parquet; spectra_peaks.parquet is supplementary + /// (additional centroid peaks for profile spectra that were peak-picked). + /// + public sealed record BinaryArrays(double[] Mz, float[] Intensity); + + private const string CurieProfileSpectrum = "MS:1000128"; + private const string CurieCentroidSpectrum = "MS:1000127"; + + // CV column name pattern. Captures the trailing "_unit_(MS|UO)_NNNN" suffix when present + // — informational only; the unit is also implicit in the column name and not load-bearing + // for Phase 2/2.5 access. Also accepts an optional Parquet list-element trailer: + // Arrow / real mzPeak files use ".list.element" embedded in the path (handled by the + // (?:\.)* group), while ParquetSharp's writer adds a trailing ".list.item" to + // declare the leaf inside a list group. Accept either trailer so files written by + // both stacks dispatch to the same column. + private static readonly Regex CvColumnRegex = new( + @"^(?[a-z_]+)(?:\.(?[a-z_]+))*\." + + @"MS_(?\d{7})_(?[a-z_]+?)" + + @"(?:_unit_(?MS|UO)_(?\d{7}))?" + + @"(?:\.list\.(?:item|element))?$", + RegexOptions.Compiled); + + // Source of the parquet entries: read in place from the .mzpeak ZIP (Stored entries) when + // possible, else extracted to a temp dir. Exactly one of _archive / _tempDir is set. + private readonly MzPeakArchive? _archive; + private readonly string? _tempDir; + + // Lazy spectra-metadata: the parquet reader stays open; columns are allocated full-height at + // construction but each row group's slice is decoded only on first access (EnsureGroupLoaded), + // so opening a file doesn't pay to read every metadata column. The two linking columns + // (spectrum.index, precursor.source_index) are read eagerly because the fan-out map needs them. + private readonly ParquetFileReader _metaReader; + private readonly Dictionary _metaCols; + private readonly int _numMetaGroups; + private readonly int[] _rgRowOffset; // cumulative row offsets, length _numMetaGroups + 1 + private readonly bool[] _metaGroupLoaded; + private readonly object _metaLock = new(); + + // Scalar columns (Phase 2) — allocated full-height, filled lazily per row group. Schema marks + // everything nullable (maxDef=2 — optional parent struct + optional column), so storage is `T?[]`. + private readonly ulong?[] _index; + private readonly string?[] _id; + private readonly double?[] _time; + private readonly int?[] _msLevel; + private readonly string?[] _representationCurie; + private readonly double?[] _scanStartTime; + private readonly string?[] _filterString; + private readonly uint?[] _instrumentConfigRef; + private readonly double?[] _ionInjectionTime; + + // Phase 2.5 — list columns (spectrum.parameters, scan.parameters). Each row holds + // a nullable array of leaf values; zipping the parallel arrays reconstructs the + // CvParam list. ParquetSharp returns null for "row had no list at all" vs an empty + // array for "list was present but empty"; we collapse both to empty list at read. + private readonly string?[]?[] _spectrumParamsName; + private readonly string?[]?[] _spectrumParamsAccession; + private readonly string?[]?[] _spectrumParamsValueString; + private readonly long?[]?[] _spectrumParamsValueInteger; + private readonly double?[]?[] _spectrumParamsValueFloat; + private readonly bool?[]?[] _spectrumParamsValueBoolean; + private readonly string?[]?[] _spectrumParamsUnit; + private readonly string?[]?[] _spectrumParamsType; + + private readonly string?[]?[] _scanParamsName; + private readonly string?[]?[] _scanParamsAccession; + private readonly string?[]?[] _scanParamsValueString; + private readonly long?[]?[] _scanParamsValueInteger; + private readonly double?[]?[] _scanParamsValueFloat; + private readonly bool?[]?[] _scanParamsValueBoolean; + private readonly string?[]?[] _scanParamsUnit; + private readonly string?[]?[] _scanParamsType; + + // Precursor scalars (Phase 2.5). + private readonly string?[] _precursorId; + private readonly double?[] _isolationTargetMz; + private readonly double?[] _isolationLowerOffset; + private readonly double?[] _isolationUpperOffset; + private readonly double?[] _collisionEnergy; + private readonly string?[] _dissociationMethod; + private readonly double?[] _selectedIonMz; + private readonly double?[] _selectedIonPeakIntensity; + private readonly long?[] _selectedIonChargeState; + + // Phase 5 — nested parameter lists. Each entity's params are seven parallel + // arrays of nullable lists per row (one list per spectrum row). Naming pattern + // matches the columnar leaf: .parameters.list.element.{name,accession, + // value.{string,integer,float,boolean},unit}. + private readonly string?[]?[] _isoParamsName; + private readonly string?[]?[] _isoParamsAccession; + private readonly string?[]?[] _isoParamsValueString; + private readonly long?[]?[] _isoParamsValueInteger; + private readonly double?[]?[] _isoParamsValueFloat; + private readonly bool?[]?[] _isoParamsValueBoolean; + private readonly string?[]?[] _isoParamsUnit; + private readonly string?[]?[] _isoParamsType; + + private readonly string?[]?[] _actParamsName; + private readonly string?[]?[] _actParamsAccession; + private readonly string?[]?[] _actParamsValueString; + private readonly long?[]?[] _actParamsValueInteger; + private readonly double?[]?[] _actParamsValueFloat; + private readonly bool?[]?[] _actParamsValueBoolean; + private readonly string?[]?[] _actParamsUnit; + private readonly string?[]?[] _actParamsType; + + private readonly string?[]?[] _siParamsName; + private readonly string?[]?[] _siParamsAccession; + private readonly string?[]?[] _siParamsValueString; + private readonly long?[]?[] _siParamsValueInteger; + private readonly double?[]?[] _siParamsValueFloat; + private readonly bool?[]?[] _siParamsValueBoolean; + private readonly string?[]?[] _siParamsUnit; + private readonly string?[]?[] _siParamsType; + + // Scan windows — per-spectrum list of (lower, upper). The element columns + // are themselves rep=1 (one list of doubles per spectrum row, where each + // element corresponds to one scan window). + private readonly string?[] _scanWindowParamsJson; + private readonly string?[] _extraScansJson; + private readonly string?[] _scanSpectrumRef; + private readonly double?[]?[] _scanWindowLower; // MS:1000501 + private readonly double?[]?[] _scanWindowUpper; // MS:1000500 + + // Phase 7 — MSData parity scalar storage. + private readonly int?[] _scanPolarity; + private readonly string?[] _spectrumTypeCurie; + private readonly long?[] _numberOfDataPoints; + private readonly long?[] _numberOfPeaks; + private readonly double?[] _basePeakMz; + private readonly double?[] _basePeakIntensity; + private readonly double?[] _totalIonCurrent; + private readonly double?[] _lowestObservedMz; + private readonly double?[] _highestObservedMz; + private readonly string?[] _spectrumDataProcessingRef; + private readonly int?[] _numberOfAuxiliaryArrays; + private readonly double?[]?[] _mzDeltaModel; + private readonly string?[] _scanCombinationCurie; + private readonly string?[]?[] _paramGroupRefs; + private readonly string?[] _spectrumAuxArraysJson; + private readonly string?[] _valueArrayType; // non-null when the value array isn't m/z (UV wavelength, …) + private readonly string?[] _valueArrayUnit; + + private readonly ulong?[] _scanSourceIndex; + private readonly double?[] _scanIonMobilityValue; + private readonly string?[] _scanIonMobilityType; + private readonly long?[] _scanPresetConfiguration; + + private readonly ulong?[] _precursorSourceIndex; + private readonly ulong?[] _precursorPrecursorIndex; + + private readonly ulong?[] _selectedIonSourceIndex; + private readonly ulong?[] _selectedIonPrecursorIndex; + private readonly double?[] _selectedIonIonMobilityValue; + private readonly string?[] _selectedIonIonMobilityType; + + // Phase 3 — binary data. Eager-load the point-layout files, bucket by spectrum_index, + // Binary data is read lazily, one row group at a time, with an LRU cache (see LazyPointLayout) — + // open stays cheap and memory is bounded to a few row groups instead of the whole file. + private readonly IPointLayout _dataLayer; // spectra_data.parquet (canonical) + private readonly IPointLayout _peaksLayer; // spectra_peaks.parquet (supplementary) + + // UV/DAD wavelength spectra. mzPeak.NET stores these in dedicated wavelength_spectra_* entries + // (the value array is wavelength, not m/z) rather than inline in spectra_metadata. When present + // they're surfaced after the MS spectra: logical index >= _msSpectrumCount routes here. + private readonly WavelengthSpectra? _wavelength; + private int _msSpectrumCount; + + // Phase 4b — chromatograms. Same point-layout convention as spectra; eager-load + // (time, intensity) by chromatogram_index. Demo file: 3431 chromatogram points + // across 105 chromatograms. + private readonly ulong?[] _chromIndex; + private readonly string?[] _chromId; + private readonly string?[] _chromTypeCurie; // MS:1000625 / MS:1000235 / MS:1000810 / ... + private readonly string?[] _chromDataProcessingRef; + private readonly string?[] _chromTimeUnitCurie; // unit of the time array (UO:0000010 / UO:0000031) + private readonly string?[] _chromIntensityUnitCurie; // unit of the intensity array (counts / % / pascal / …) + private readonly string?[]?[] _chromParamsName; + private readonly string?[]?[] _chromParamsAccession; + private readonly string?[]?[] _chromParamsValueString; + private readonly long?[]?[] _chromParamsValueInteger; + private readonly double?[]?[] _chromParamsValueFloat; + private readonly bool?[]?[] _chromParamsValueBoolean; + private readonly string?[]?[] _chromParamsUnit; + private readonly string?[]?[] _chromParamsType; + private readonly string?[] _chromAuxArraysJson; + private readonly IPointLayout _chromData; + + /// A per-parent binary point layer (spectra/chromatogram data or peaks). Two on-disk + /// shapes implement this: the canonical row-per-point and the + /// chunked (mzPeak.NET's compressed variant). The optional + /// spacingModel is the parent's mz_delta_model polynomial, used only by the chunked layer + /// to fill delta-encoding seam nulls. + private interface IPointLayout : IDisposable + { + (double[] Value, float[] Intensity)? Get(long parentIndex, double[]? spacingModel = null); + } + + /// Chromatogram metadata + data. + public sealed record ChromatogramDescription( + ulong Index, + string Id, + string? ChromatogramTypeCurie, + string? DataProcessingRef, + string? TimeUnitCurie = null, + string? IntensityUnitCurie = null, + IReadOnlyList? Parameters = null, + IReadOnlyList? AuxArrays = null); + + /// Time + intensity arrays for one chromatogram. + public sealed record ChromatogramArrays(double[] Time, float[] Intensity); + + /// File-level metadata (the Parquet KV JSON blocks). Lazily parsed + /// the first time it's accessed. + public FileMetadata FileMetadata => _fileMetadata.Value; + private readonly Lazy _fileMetadata; + + public int SpectrumCount { get; private set; } + public int ChromatogramCount { get; } + + // Multi-precursor fan-out: every parquet row contributes to one spectrum + // but a given spectrum may span N rows (one per precursor). _primaryRowOf + // maps a logical spectrum index → the row carrying its spectrum/scan-level + // fields (the row where spectrum.index is non-null). _precursorRowsBySpec + // lists every row whose precursor.source_index matches that spectrum. + private readonly int[] _primaryRowOfSpectrum; + private readonly int[][] _precursorRowsBySpectrum; + + public MzPeakReader(string path) + { + // Prefer reading parquet entries in place from the ZIP; extract only if that isn't possible. + _archive = MzPeakArchive.TryOpen(path); + _tempDir = _archive is null ? ExtractZip(path) : null; + + _metaReader = OpenParquet("spectra_metadata.parquet")!; + _metaCols = BuildColumnIndex(_metaReader.FileMetaData.Schema); + _numMetaGroups = _metaReader.FileMetaData.NumRowGroups; + _rgRowOffset = new int[_numMetaGroups + 1]; + for (int g = 0; g < _numMetaGroups; g++) + { + using var rgr = _metaReader.RowGroup(g); + _rgRowOffset[g + 1] = _rgRowOffset[g] + checked((int)rgr.MetaData.NumRows); + } + int totalRows = _rgRowOffset[_numMetaGroups]; + SpectrumCount = totalRows; + _metaGroupLoaded = new bool[_numMetaGroups]; + + // Allocate the full-height column arrays once; row-group slices are filled lazily on first + // access (EnsureGroupLoaded), so opening the file decodes no per-spectrum columns. + _index = new ulong?[totalRows]; + _id = new string?[totalRows]; + _time = new double?[totalRows]; + _msLevel = new int?[totalRows]; + _representationCurie = new string?[totalRows]; + _scanStartTime = new double?[totalRows]; + _filterString = new string?[totalRows]; + _instrumentConfigRef = new uint?[totalRows]; + _ionInjectionTime = new double?[totalRows]; + _spectrumParamsName = new string?[totalRows][]; + _spectrumParamsAccession = new string?[totalRows][]; + _spectrumParamsValueString = new string?[totalRows][]; + _spectrumParamsValueInteger = new long?[totalRows][]; + _spectrumParamsValueFloat = new double?[totalRows][]; + _spectrumParamsValueBoolean = new bool?[totalRows][]; + _spectrumParamsUnit = new string?[totalRows][]; + _spectrumParamsType = new string?[totalRows][]; + _scanParamsName = new string?[totalRows][]; + _scanParamsAccession = new string?[totalRows][]; + _scanParamsValueString = new string?[totalRows][]; + _scanParamsValueInteger = new long?[totalRows][]; + _scanParamsValueFloat = new double?[totalRows][]; + _scanParamsValueBoolean = new bool?[totalRows][]; + _scanParamsUnit = new string?[totalRows][]; + _scanParamsType = new string?[totalRows][]; + _precursorId = new string?[totalRows]; + _isolationTargetMz = new double?[totalRows]; + _isolationLowerOffset = new double?[totalRows]; + _isolationUpperOffset = new double?[totalRows]; + _collisionEnergy = new double?[totalRows]; + _dissociationMethod = new string?[totalRows]; + _selectedIonMz = new double?[totalRows]; + _selectedIonPeakIntensity = new double?[totalRows]; + _selectedIonChargeState = new long?[totalRows]; + _isoParamsName = new string?[totalRows][]; + _isoParamsAccession = new string?[totalRows][]; + _isoParamsValueString = new string?[totalRows][]; + _isoParamsValueInteger = new long?[totalRows][]; + _isoParamsValueFloat = new double?[totalRows][]; + _isoParamsValueBoolean = new bool?[totalRows][]; + _isoParamsUnit = new string?[totalRows][]; + _isoParamsType = new string?[totalRows][]; + _actParamsName = new string?[totalRows][]; + _actParamsAccession = new string?[totalRows][]; + _actParamsValueString = new string?[totalRows][]; + _actParamsValueInteger = new long?[totalRows][]; + _actParamsValueFloat = new double?[totalRows][]; + _actParamsValueBoolean = new bool?[totalRows][]; + _actParamsUnit = new string?[totalRows][]; + _actParamsType = new string?[totalRows][]; + _siParamsName = new string?[totalRows][]; + _siParamsAccession = new string?[totalRows][]; + _siParamsValueString = new string?[totalRows][]; + _siParamsValueInteger = new long?[totalRows][]; + _siParamsValueFloat = new double?[totalRows][]; + _siParamsValueBoolean = new bool?[totalRows][]; + _siParamsUnit = new string?[totalRows][]; + _siParamsType = new string?[totalRows][]; + _scanWindowLower = new double?[totalRows][]; + _scanWindowUpper = new double?[totalRows][]; + _scanWindowParamsJson = new string?[totalRows]; + _extraScansJson = new string?[totalRows]; + _scanSpectrumRef = new string?[totalRows]; + _scanPolarity = new int?[totalRows]; + _spectrumTypeCurie = new string?[totalRows]; + _numberOfDataPoints = new long?[totalRows]; + _numberOfPeaks = new long?[totalRows]; + _basePeakMz = new double?[totalRows]; + _basePeakIntensity = new double?[totalRows]; + _totalIonCurrent = new double?[totalRows]; + _lowestObservedMz = new double?[totalRows]; + _highestObservedMz = new double?[totalRows]; + _spectrumDataProcessingRef = new string?[totalRows]; + _numberOfAuxiliaryArrays = new int?[totalRows]; + _mzDeltaModel = new double?[totalRows][]; + _scanCombinationCurie = new string?[totalRows]; + _paramGroupRefs = new string?[totalRows][]; + _spectrumAuxArraysJson = new string?[totalRows]; + _valueArrayType = new string?[totalRows]; + _valueArrayUnit = new string?[totalRows]; + _scanSourceIndex = new ulong?[totalRows]; + _scanIonMobilityValue = new double?[totalRows]; + _scanIonMobilityType = new string?[totalRows]; + _scanPresetConfiguration = new long?[totalRows]; + _precursorSourceIndex = new ulong?[totalRows]; + _precursorPrecursorIndex = new ulong?[totalRows]; + _selectedIonSourceIndex = new ulong?[totalRows]; + _selectedIonPrecursorIndex = new ulong?[totalRows]; + _selectedIonIonMobilityValue = new double?[totalRows]; + _selectedIonIonMobilityType = new string?[totalRows]; + + // Read eagerly: the two linking columns (fan-out map) and id (SpectrumIdentity list, built + // at open). Everything else is decoded lazily per row group on first GetSpectrumDescription. + ReadColumnAllGroups(_index, (rg, n) => ReadNullableValue(rg, _metaCols, "spectrum.index", n)); + ReadColumnAllGroups(_precursorSourceIndex, (rg, n) => ReadNullableValue(rg, _metaCols, "precursor.source_index", n)); + ReadColumnAllGroups(_id, (rg, n) => ReadNullableString(rg, _metaCols, "spectrum.id", n)); + + // Multi-precursor fan-out. A spectrum with N precursors writes N parquet + // rows: row 0 carries spectrum + scan + precursor[0] + selected_ion[0]; + // rows 1..N-1 carry only precursor[k] + selected_ion[k] (spectrum/scan + // groups are null). Primary rows are those where spectrum.index is set. + int rowCount = SpectrumCount; + var primary = new List(rowCount); + for (int r = 0; r < rowCount; r++) if (_index[r].HasValue) primary.Add(r); + _primaryRowOfSpectrum = primary.ToArray(); + SpectrumCount = _primaryRowOfSpectrum.Length; + + var precGroups = new Dictionary>(); + for (int r = 0; r < rowCount; r++) + { + if (_precursorSourceIndex[r] is ulong s) + { + if (!precGroups.TryGetValue(s, out var list)) precGroups[s] = list = new List(); + list.Add(r); + } + } + _precursorRowsBySpectrum = new int[SpectrumCount][]; + for (int i = 0; i < SpectrumCount; i++) + { + var sIdx = _index[_primaryRowOfSpectrum[i]]!.Value; + _precursorRowsBySpectrum[i] = precGroups.TryGetValue(sIdx, out var rs) ? rs.ToArray() : Array.Empty(); + } + + // Phase 3 — open the lazy point-layout readers (reads only the row-group range KV). + _dataLayer = OpenPointLayout("spectra_data.parquet"); + _peaksLayer = OpenPointLayout("spectra_peaks.parquet"); + + // Phase 4a — file-level metadata is lazily parsed from the same parquet + // file's KV (the writer puts it there once per file). Snapshot the KV now + // so we don't have to reopen the file when callers ask for it. + var kvSnapshot = _metaReader.FileMetaData.KeyValueMetadata.ToDictionary(p => p.Key, p => p.Value); + _fileMetadata = new Lazy(() => FileMetadataDeserializer.Parse(kvSnapshot)); + + // Phase 4b — chromatograms. Same column-dispatcher pattern as spectra, + // separate parquet files. Phase 4b reads scalar columns (id, type CURIE, + // data_processing_ref) + point-layout (time + intensity); precursor / + // selected_ion under chromatograms are deferred. + if (HasEntry("chromatograms_metadata.parquet")) + { + using var chromReader = OpenParquet("chromatograms_metadata.parquet")!; + ChromatogramCount = checked((int)chromReader.FileMetaData.NumRows); + var chromCols = BuildColumnIndex(chromReader.FileMetaData.Schema); + using var chromRg = chromReader.RowGroup(0); + _chromIndex = ReadNullableValue(chromRg, chromCols, "chromatogram.index", ChromatogramCount); + _chromId = ReadNullableString(chromRg, chromCols, "chromatogram.id", ChromatogramCount); + _chromTypeCurie = ReadNullableString(chromRg, chromCols, "MS:1000626", ChromatogramCount); + _chromDataProcessingRef = ReadNullableString(chromRg, chromCols, "chromatogram.data_processing_ref", ChromatogramCount); + _chromTimeUnitCurie = ReadNullableString(chromRg, chromCols, "MS:1000595", ChromatogramCount); + _chromIntensityUnitCurie = ReadNullableString(chromRg, chromCols, "MS:1000515", ChromatogramCount); + _chromParamsName = ReadNullableStringList(chromRg, chromCols, "chromatogram.parameters.list.element.name", ChromatogramCount); + _chromParamsAccession = ReadNullableStringList(chromRg, chromCols, "chromatogram.parameters.list.element.accession", ChromatogramCount); + _chromParamsValueString = ReadNullableStringList(chromRg, chromCols, "chromatogram.parameters.list.element.value.string", ChromatogramCount); + _chromParamsValueInteger = ReadNullableValueList(chromRg, chromCols, "chromatogram.parameters.list.element.value.integer", ChromatogramCount); + _chromParamsValueFloat = ReadNullableValueList(chromRg, chromCols, "chromatogram.parameters.list.element.value.float", ChromatogramCount); + _chromParamsValueBoolean = ReadNullableValueList(chromRg, chromCols, "chromatogram.parameters.list.element.value.boolean", ChromatogramCount); + _chromParamsUnit = ReadNullableStringList(chromRg, chromCols, "chromatogram.parameters.list.element.unit", ChromatogramCount); + _chromParamsType = ReadNullableStringList(chromRg, chromCols, "chromatogram.parameters.list.element.type", ChromatogramCount); + _chromAuxArraysJson = ReadNullableString(chromRg, chromCols, "chromatogram.auxiliary_arrays", ChromatogramCount); + } + else + { + ChromatogramCount = 0; + _chromIndex = Array.Empty(); + _chromId = Array.Empty(); + _chromTypeCurie = Array.Empty(); + _chromDataProcessingRef = Array.Empty(); + _chromTimeUnitCurie = Array.Empty(); + _chromIntensityUnitCurie = Array.Empty(); + _chromParamsName = Array.Empty(); + _chromParamsAccession = Array.Empty(); + _chromParamsValueString = Array.Empty(); + _chromParamsValueInteger = Array.Empty(); + _chromParamsValueFloat = Array.Empty(); + _chromParamsValueBoolean = Array.Empty(); + _chromParamsUnit = Array.Empty(); + _chromParamsType = Array.Empty(); + _chromAuxArraysJson = Array.Empty(); + } + _chromData = OpenPointLayout("chromatograms_data.parquet"); + + // UV/DAD wavelength spectra live in separate parquet entries (mzPeak.NET). Load them as a + // secondary spectrum table appended after the MS spectra. + _msSpectrumCount = SpectrumCount; + if (HasEntry("wavelength_spectra_metadata.parquet")) + { + _wavelength = new WavelengthSpectra( + OpenParquet("wavelength_spectra_metadata.parquet")!, + OpenParquet("wavelength_spectra_data.parquet")); + SpectrumCount = _msSpectrumCount + _wavelength.Count; + } + } + + /// Open a parquet entry — in place from the ZIP archive, or from the extracted temp dir. + private ParquetFileReader? OpenParquet(string name) + { + if (_archive is not null) return _archive.OpenParquet(name); + string p = System.IO.Path.Combine(_tempDir!, name); + return File.Exists(p) ? new ParquetFileReader(p) : null; + } + + private bool HasEntry(string name) => + _archive?.HasEntry(name) ?? File.Exists(System.IO.Path.Combine(_tempDir!, name)); + + /// Open a binary point-layout entry, selecting the chunked decoder when the data uses the + /// chunked buffer format and the canonical row-per-point reader otherwise. A missing entry yields + /// an empty layout that returns null for every lookup. + private IPointLayout OpenPointLayout(string name) => MakePointLayout(OpenParquet(name)); + + private static IPointLayout MakePointLayout(ParquetFileReader? reader) + { + if (reader is null) return new LazyPointLayout(null); + return ChunkedPointLayout.IsChunked(reader) + ? new ChunkedPointLayout(reader) + : new LazyPointLayout(reader); + } + + /// Each row group's [min,max] for an integer parent-index column, preferring the (free) + /// Parquet column-chunk statistics and only reading the column when a group has no usable stats. + /// Empty/unreadable groups get (-1,-1). Ranges may overlap; callers merge all covering groups. + private static (long First, long Last)[] DeriveRangesFromColumn(ParquetFileReader reader, int idxCol) + { + int groups = reader.FileMetaData.NumRowGroups; + var derived = new List<(long, long)>(groups); + for (int g = 0; g < groups; g++) + { + using var rgr = reader.RowGroup(g); + int rows = checked((int)rgr.MetaData.NumRows); + if (rows == 0) { derived.Add((-1, -1)); continue; } + + using var cc = rgr.MetaData.GetColumnChunkMetaData(idxCol); + if (TryStatsRange(cc.Statistics, out long smin, out long smax)) + { + derived.Add((smin, smax)); + continue; + } + + var idx = new ulong?[rows]; + using (var c = rgr.Column(idxCol).LogicalReader()) c.ReadBatch(idx.AsSpan()); + long min = long.MaxValue, max = long.MinValue; + foreach (var u in idx) if (u is { } v) { if ((long)v < min) min = (long)v; if ((long)v > max) max = (long)v; } + derived.Add(min <= max ? (min, max) : (-1, -1)); + } + return derived.ToArray(); + } + + /// Extract a row group's [min,max] for an index column from Parquet statistics, if set. + /// Index columns are integers; the concrete Statistics<T> varies by physical width and + /// signedness, so match each. Returns false when stats are absent (caller reads the column). + private static bool TryStatsRange(Statistics? stats, out long min, out long max) + { + min = 0; max = 0; + if (stats is null || !stats.HasMinMax) return false; + switch (stats) + { + case Statistics s: min = s.Min; max = s.Max; return true; + case Statistics s: min = s.Min; max = s.Max; return true; + case Statistics s: min = s.Min; max = s.Max; return true; + case Statistics s: min = (long)s.Min; max = (long)s.Max; return true; + default: return false; + } + } + + /// Get a chromatogram description by row index. + public ChromatogramDescription GetChromatogramDescription(int rowIndex) + { + if ((uint)rowIndex >= (uint)ChromatogramCount) + throw new ArgumentOutOfRangeException(nameof(rowIndex)); + return new ChromatogramDescription( + Index: _chromIndex[rowIndex] ?? 0, + Id: _chromId[rowIndex] ?? string.Empty, + ChromatogramTypeCurie: _chromTypeCurie[rowIndex], + DataProcessingRef: _chromDataProcessingRef[rowIndex], + TimeUnitCurie: _chromTimeUnitCurie[rowIndex], + IntensityUnitCurie: _chromIntensityUnitCurie[rowIndex], + Parameters: ZipParams( + _chromParamsName[rowIndex], _chromParamsAccession[rowIndex], + _chromParamsValueString[rowIndex], _chromParamsValueInteger[rowIndex], + _chromParamsValueFloat[rowIndex], _chromParamsValueBoolean[rowIndex], + _chromParamsUnit[rowIndex], _chromParamsType[rowIndex]), + AuxArrays: AuxiliaryArrays.Parse(_chromAuxArraysJson[rowIndex])); + } + + /// Get a chromatogram's (time, intensity) arrays. Returns null when + /// no point rows reference this chromatogram in chromatograms_data.parquet. + public ChromatogramArrays? GetChromatogramData(int rowIndex) + { + if ((uint)rowIndex >= (uint)ChromatogramCount) + throw new ArgumentOutOfRangeException(nameof(rowIndex)); + // Key on the stored chromatogram.index (see GetSpectrumData for the rationale). + long key = (long)(_chromIndex[rowIndex] ?? (ulong)rowIndex); + if (_chromData.Get(key) is { } d) return new ChromatogramArrays(d.Value, d.Intensity); + return null; + } + + private void ReadColumnAllGroups(T[] dest, Func read) + { + for (int g = 0; g < _numMetaGroups; g++) + { + using var rg = _metaReader.RowGroup(g); + int n = _rgRowOffset[g + 1] - _rgRowOffset[g]; + Array.Copy(read(rg, n), 0, dest, _rgRowOffset[g], n); + } + } + + private static void CopyInto(T[] dest, T[] src, int off) => Array.Copy(src, 0, dest, off, src.Length); + + private int GroupOfRow(int row) + { + int lo = 0, hi = _numMetaGroups - 1; + while (lo < hi) + { + int mid = (lo + hi + 1) >> 1; + if (_rgRowOffset[mid] <= row) lo = mid; else hi = mid - 1; + } + return lo; + } + + private void EnsureGroupLoaded(int g) + { + if (_metaGroupLoaded[g]) return; + lock (_metaLock) + { + if (_metaGroupLoaded[g]) return; + LoadMetaGroup(g); + _metaGroupLoaded[g] = true; + } + } + + /// Decode every spectra-metadata column for one row group into its slice of the full arrays. + private void LoadMetaGroup(int g) + { + using var rg = _metaReader.RowGroup(g); + int n = _rgRowOffset[g + 1] - _rgRowOffset[g]; + int off = _rgRowOffset[g]; + + CopyInto(_id, ReadNullableString(rg, _metaCols, "spectrum.id", n), off); + CopyInto(_time, ReadNullableValue(rg, _metaCols, "spectrum.time", n), off); + CopyInto(_msLevel, ReadNullableInt8(rg, _metaCols, "MS:1000511", n), off); + CopyInto(_representationCurie, ReadNullableString(rg, _metaCols, "MS:1000525", n), off); + CopyInto(_scanStartTime, ReadNullableValue(rg, _metaCols, "MS:1000016", n), off); + CopyInto(_filterString, ReadNullableString(rg, _metaCols, "MS:1000512", n), off); + CopyInto(_instrumentConfigRef, ReadNullableValue(rg, _metaCols, "scan.instrument_configuration_ref", n), off); + CopyInto(_ionInjectionTime, ReadNullableValue(rg, _metaCols, "MS:1000927", n), off); + + CopyInto(_spectrumParamsName, ReadNullableStringList(rg, _metaCols, "spectrum.parameters.list.element.name", n), off); + CopyInto(_spectrumParamsAccession, ReadNullableStringList(rg, _metaCols, "spectrum.parameters.list.element.accession", n), off); + CopyInto(_spectrumParamsValueString, ReadNullableStringList(rg, _metaCols, "spectrum.parameters.list.element.value.string", n), off); + CopyInto(_spectrumParamsValueInteger, ReadNullableValueList(rg, _metaCols, "spectrum.parameters.list.element.value.integer", n), off); + CopyInto(_spectrumParamsValueFloat, ReadNullableValueList(rg, _metaCols, "spectrum.parameters.list.element.value.float", n), off); + CopyInto(_spectrumParamsValueBoolean, ReadNullableValueList(rg, _metaCols, "spectrum.parameters.list.element.value.boolean", n), off); + CopyInto(_spectrumParamsUnit, ReadNullableStringList(rg, _metaCols, "spectrum.parameters.list.element.unit", n), off); + CopyInto(_spectrumParamsType, ReadNullableStringList(rg, _metaCols, "spectrum.parameters.list.element.type", n), off); + + CopyInto(_scanParamsName, ReadNullableStringList(rg, _metaCols, "scan.parameters.list.element.name", n), off); + CopyInto(_scanParamsAccession, ReadNullableStringList(rg, _metaCols, "scan.parameters.list.element.accession", n), off); + CopyInto(_scanParamsValueString, ReadNullableStringList(rg, _metaCols, "scan.parameters.list.element.value.string", n), off); + CopyInto(_scanParamsValueInteger, ReadNullableValueList(rg, _metaCols, "scan.parameters.list.element.value.integer", n), off); + CopyInto(_scanParamsValueFloat, ReadNullableValueList(rg, _metaCols, "scan.parameters.list.element.value.float", n), off); + CopyInto(_scanParamsValueBoolean, ReadNullableValueList(rg, _metaCols, "scan.parameters.list.element.value.boolean", n), off); + CopyInto(_scanParamsUnit, ReadNullableStringList(rg, _metaCols, "scan.parameters.list.element.unit", n), off); + CopyInto(_scanParamsType, ReadNullableStringList(rg, _metaCols, "scan.parameters.list.element.type", n), off); + + CopyInto(_precursorId, ReadNullableString(rg, _metaCols, "precursor.precursor_id", n), off); + CopyInto(_isolationTargetMz, ReadNullableValue(rg, _metaCols, "MS:1000827", n), off); + CopyInto(_isolationLowerOffset, ReadNullableValue(rg, _metaCols, "MS:1000828", n), off); + CopyInto(_isolationUpperOffset, ReadNullableValue(rg, _metaCols, "MS:1000829", n), off); + CopyInto(_collisionEnergy, ReadNullableValue(rg, _metaCols, "MS:1000045", n), off); + CopyInto(_dissociationMethod, ReadNullableString(rg, _metaCols, "MS:1000044", n), off); + CopyInto(_selectedIonMz, ReadNullableValue(rg, _metaCols, "MS:1000744", n), off); + CopyInto(_selectedIonPeakIntensity, ReadNullableValue(rg, _metaCols, "MS:1000042", n), off); + CopyInto(_selectedIonChargeState, ReadNullableValue(rg, _metaCols, "MS:1000041", n), off); + + CopyInto(_isoParamsName, ReadNullableStringList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.name", n), off); + CopyInto(_isoParamsAccession, ReadNullableStringList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.accession", n), off); + CopyInto(_isoParamsValueString, ReadNullableStringList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.value.string", n), off); + CopyInto(_isoParamsValueInteger, ReadNullableValueList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.value.integer", n), off); + CopyInto(_isoParamsValueFloat, ReadNullableValueList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.value.float", n), off); + CopyInto(_isoParamsValueBoolean, ReadNullableValueList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.value.boolean", n), off); + CopyInto(_isoParamsUnit, ReadNullableStringList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.unit", n), off); + CopyInto(_isoParamsType, ReadNullableStringList(rg, _metaCols, "precursor.isolation_window.parameters.list.element.type", n), off); + + CopyInto(_actParamsName, ReadNullableStringList(rg, _metaCols, "precursor.activation.parameters.list.element.name", n), off); + CopyInto(_actParamsAccession, ReadNullableStringList(rg, _metaCols, "precursor.activation.parameters.list.element.accession", n), off); + CopyInto(_actParamsValueString, ReadNullableStringList(rg, _metaCols, "precursor.activation.parameters.list.element.value.string", n), off); + CopyInto(_actParamsValueInteger, ReadNullableValueList(rg, _metaCols, "precursor.activation.parameters.list.element.value.integer", n), off); + CopyInto(_actParamsValueFloat, ReadNullableValueList(rg, _metaCols, "precursor.activation.parameters.list.element.value.float", n), off); + CopyInto(_actParamsValueBoolean, ReadNullableValueList(rg, _metaCols, "precursor.activation.parameters.list.element.value.boolean", n), off); + CopyInto(_actParamsUnit, ReadNullableStringList(rg, _metaCols, "precursor.activation.parameters.list.element.unit", n), off); + CopyInto(_actParamsType, ReadNullableStringList(rg, _metaCols, "precursor.activation.parameters.list.element.type", n), off); + + CopyInto(_siParamsName, ReadNullableStringList(rg, _metaCols, "selected_ion.parameters.list.element.name", n), off); + CopyInto(_siParamsAccession, ReadNullableStringList(rg, _metaCols, "selected_ion.parameters.list.element.accession", n), off); + CopyInto(_siParamsValueString, ReadNullableStringList(rg, _metaCols, "selected_ion.parameters.list.element.value.string", n), off); + CopyInto(_siParamsValueInteger, ReadNullableValueList(rg, _metaCols, "selected_ion.parameters.list.element.value.integer", n), off); + CopyInto(_siParamsValueFloat, ReadNullableValueList(rg, _metaCols, "selected_ion.parameters.list.element.value.float", n), off); + CopyInto(_siParamsValueBoolean, ReadNullableValueList(rg, _metaCols, "selected_ion.parameters.list.element.value.boolean", n), off); + CopyInto(_siParamsUnit, ReadNullableStringList(rg, _metaCols, "selected_ion.parameters.list.element.unit", n), off); + CopyInto(_siParamsType, ReadNullableStringList(rg, _metaCols, "selected_ion.parameters.list.element.type", n), off); + + CopyInto(_scanWindowLower, ReadNullableValueList(rg, _metaCols, "MS:1000501", n), off); + CopyInto(_scanWindowUpper, ReadNullableValueList(rg, _metaCols, "MS:1000500", n), off); + CopyInto(_scanWindowParamsJson, ReadNullableString(rg, _metaCols, "scan.scan_window_params", n), off); + CopyInto(_extraScansJson, ReadNullableString(rg, _metaCols, "scan.extra_scans", n), off); + CopyInto(_scanSpectrumRef, ReadNullableString(rg, _metaCols, "scan.spectrum_ref", n), off); + + CopyInto(_scanPolarity, ReadNullableInt8(rg, _metaCols, "MS:1000465", n), off); + CopyInto(_spectrumTypeCurie, ReadNullableString(rg, _metaCols, "MS:1000559", n), off); + CopyInto(_numberOfDataPoints, ReadNullableValue(rg, _metaCols, "MS:1003060", n), off); + CopyInto(_numberOfPeaks, ReadNullableValue(rg, _metaCols, "MS:1003059", n), off); + CopyInto(_basePeakMz, ReadNullableValue(rg, _metaCols, "MS:1000504", n), off); + CopyInto(_basePeakIntensity, ReadNullableValue(rg, _metaCols, "MS:1000505", n), off); + CopyInto(_totalIonCurrent, ReadNullableValue(rg, _metaCols, "MS:1000285", n), off); + CopyInto(_lowestObservedMz, ReadNullableValue(rg, _metaCols, "MS:1000528", n), off); + CopyInto(_highestObservedMz, ReadNullableValue(rg, _metaCols, "MS:1000527", n), off); + CopyInto(_spectrumDataProcessingRef, ReadNullableString(rg, _metaCols, "spectrum.data_processing_ref", n), off); + CopyInto(_numberOfAuxiliaryArrays, ReadNullableValueAsInt(rg, _metaCols, "spectrum.number_of_auxiliary_arrays", n), off); + + CopyInto(_mzDeltaModel, ReadNullableValueList(rg, _metaCols, "spectrum.mz_delta_model.list.element", n), off); + CopyInto(_scanCombinationCurie, ReadNullableString(rg, _metaCols, "MS:1000570", n), off); + CopyInto(_paramGroupRefs, ReadNullableStringList(rg, _metaCols, "spectrum.param_group_refs.list.element", n), off); + CopyInto(_spectrumAuxArraysJson, ReadNullableString(rg, _metaCols, "spectrum.auxiliary_arrays", n), off); + CopyInto(_valueArrayType, ReadNullableString(rg, _metaCols, "spectrum.value_array_type", n), off); + CopyInto(_valueArrayUnit, ReadNullableString(rg, _metaCols, "spectrum.value_array_unit", n), off); + + CopyInto(_scanSourceIndex, ReadNullableValue(rg, _metaCols, "scan.source_index", n), off); + CopyInto(_scanIonMobilityValue, ReadNullableValue(rg, _metaCols, "scan.ion_mobility_value", n), off); + CopyInto(_scanIonMobilityType, ReadNullableString(rg, _metaCols, "scan.ion_mobility_type", n), off); + CopyInto(_scanPresetConfiguration, ReadNullableValue(rg, _metaCols, "MS:1000616", n), off); + + CopyInto(_precursorPrecursorIndex, ReadNullableValue(rg, _metaCols, "precursor.precursor_index", n), off); + CopyInto(_selectedIonSourceIndex, ReadNullableValue(rg, _metaCols, "selected_ion.source_index", n), off); + CopyInto(_selectedIonPrecursorIndex, ReadNullableValue(rg, _metaCols, "selected_ion.precursor_index", n), off); + CopyInto(_selectedIonIonMobilityValue, ReadNullableValue(rg, _metaCols, "selected_ion.ion_mobility_value", n), off); + CopyInto(_selectedIonIonMobilityType, ReadNullableString(rg, _metaCols, "selected_ion.ion_mobility_type", n), off); + } + + /// The spectrum's id without forcing a lazy metadata-group load (id is read eagerly at open). + public string GetSpectrumId(int spectrumIndex) + { + if ((uint)spectrumIndex >= (uint)SpectrumCount) + throw new ArgumentOutOfRangeException(nameof(spectrumIndex)); + if (spectrumIndex >= _msSpectrumCount) + return _wavelength!.GetId(spectrumIndex - _msSpectrumCount); + return _id[_primaryRowOfSpectrum[spectrumIndex]] ?? string.Empty; + } + + public SpectrumDescription GetSpectrumDescription(int spectrumIndex) + { + if ((uint)spectrumIndex >= (uint)SpectrumCount) + throw new ArgumentOutOfRangeException(nameof(spectrumIndex)); + if (spectrumIndex >= _msSpectrumCount) + return _wavelength!.GetDescription(spectrumIndex - _msSpectrumCount); + + int row = _primaryRowOfSpectrum[spectrumIndex]; + + // Lazily decode the row group(s) holding this spectrum's rows before reading columns. A + // spectrum's fan-out rows are kept in one group by the writer, so this is typically one group. + EnsureGroupLoaded(GroupOfRow(row)); + foreach (var pr in _precursorRowsBySpectrum[spectrumIndex]) EnsureGroupLoaded(GroupOfRow(pr)); + + var curie = _representationCurie[row]; + var spectrumParams = ZipParams( + _spectrumParamsName[row], _spectrumParamsAccession[row], + _spectrumParamsValueString[row], _spectrumParamsValueInteger[row], + _spectrumParamsValueFloat[row], _spectrumParamsValueBoolean[row], + _spectrumParamsUnit[row], _spectrumParamsType[row]); + var scanParams = ZipParams( + _scanParamsName[row], _scanParamsAccession[row], + _scanParamsValueString[row], _scanParamsValueInteger[row], + _scanParamsValueFloat[row], _scanParamsValueBoolean[row], + _scanParamsUnit[row], _scanParamsType[row]); + + var scanWindows = ZipScanWindows(_scanWindowLower[row], _scanWindowUpper[row]); + + ScanInfo? scan = (_scanStartTime[row] is null + && _filterString[row] is null + && _instrumentConfigRef[row] is null + && _ionInjectionTime[row] is null + && scanParams.Count == 0 + && scanWindows.Count == 0 + && _extraScansJson[row] is null + && _scanSpectrumRef[row] is null) + ? null + : new ScanInfo( + StartTime: _scanStartTime[row], + FilterString: _filterString[row], + InstrumentConfigurationRef: _instrumentConfigRef[row], + IonInjectionTime: _ionInjectionTime[row], + Parameters: scanParams, + ScanWindows: scanWindows, + SourceIndex: _scanSourceIndex[row], + IonMobilityValue: _scanIonMobilityValue[row], + IonMobilityTypeCurie: _scanIonMobilityType[row], + PresetScanConfiguration: _scanPresetConfiguration[row], + ScanWindowParamsJson: _scanWindowParamsJson[row], + ExtraScansJson: _extraScansJson[row], + SpectrumRef: _scanSpectrumRef[row]); + + // Gather every precursor row belonging to this spectrum. The fan-out + // writer guarantees one parquet row per precursor; rows that don't + // populate any precursor field (precursor.source_index null) aren't + // listed here, so empty Precursors == "spectrum has no precursors". + var precursors = new List(_precursorRowsBySpectrum[spectrumIndex].Length); + foreach (var pRow in _precursorRowsBySpectrum[spectrumIndex]) + { + var isoParams = ZipParams( + _isoParamsName[pRow], _isoParamsAccession[pRow], + _isoParamsValueString[pRow], _isoParamsValueInteger[pRow], + _isoParamsValueFloat[pRow], _isoParamsValueBoolean[pRow], + _isoParamsUnit[pRow], _isoParamsType[pRow]); + var actParams = ZipParams( + _actParamsName[pRow], _actParamsAccession[pRow], + _actParamsValueString[pRow], _actParamsValueInteger[pRow], + _actParamsValueFloat[pRow], _actParamsValueBoolean[pRow], + _actParamsUnit[pRow], _actParamsType[pRow]); + var siParams = ZipParams( + _siParamsName[pRow], _siParamsAccession[pRow], + _siParamsValueString[pRow], _siParamsValueInteger[pRow], + _siParamsValueFloat[pRow], _siParamsValueBoolean[pRow], + _siParamsUnit[pRow], _siParamsType[pRow]); + + var isolation = (_isolationTargetMz[pRow], _isolationLowerOffset[pRow], _isolationUpperOffset[pRow], isoParams.Count) switch + { + (null, null, null, 0) => null, + _ => new IsolationWindow(_isolationTargetMz[pRow], _isolationLowerOffset[pRow], _isolationUpperOffset[pRow], isoParams), + }; + var activation = (_collisionEnergy[pRow], _dissociationMethod[pRow], actParams.Count) switch + { + (null, null, 0) => null, + _ => new Activation(_collisionEnergy[pRow], _dissociationMethod[pRow], actParams), + }; + var selectedIon = (_selectedIonMz[pRow], _selectedIonPeakIntensity[pRow], _selectedIonChargeState[pRow], siParams.Count) switch + { + (null, null, null, 0) => null, + _ => new SelectedIonInfo( + Mz: _selectedIonMz[pRow], + PeakIntensity: _selectedIonPeakIntensity[pRow], + ChargeState: _selectedIonChargeState[pRow], + Parameters: siParams, + SourceIndex: _selectedIonSourceIndex[pRow], + PrecursorIndex: _selectedIonPrecursorIndex[pRow], + IonMobilityValue: _selectedIonIonMobilityValue[pRow], + IonMobilityTypeCurie: _selectedIonIonMobilityType[pRow]), + }; + + if (_precursorId[pRow] is null && isolation is null && activation is null && selectedIon is null) + continue; + + precursors.Add(new PrecursorInfo( + PrecursorId: _precursorId[pRow], + IsolationWindow: isolation, + Activation: activation, + SelectedIon: selectedIon, + SourceIndex: _precursorSourceIndex[pRow], + PrecursorIndex: _precursorPrecursorIndex[pRow])); + } + + IReadOnlyList? mzDeltaModel = null; + if (_mzDeltaModel[row] is { } modelArr && modelArr.Length > 0) + mzDeltaModel = modelArr.Where(d => d.HasValue).Select(d => d!.Value).ToArray(); + + IReadOnlyList? paramGroupRefs = null; + if (_paramGroupRefs[row] is { } refArr && refArr.Length > 0) + paramGroupRefs = refArr.Where(s => s is not null).Select(s => s!).ToArray(); + + return new SpectrumDescription( + Index: _index[row] ?? 0, + Id: _id[row] ?? string.Empty, + Time: _time[row] ?? 0.0, + MsLevel: _msLevel[row], + IsProfile: curie == CurieProfileSpectrum, + IsCentroid: curie == CurieCentroidSpectrum, + RepresentationCurie: curie, + Scan: scan, + Precursors: precursors, + Parameters: spectrumParams, + ScanPolarity: _scanPolarity[row], + SpectrumTypeCurie: _spectrumTypeCurie[row], + NumberOfDataPoints: _numberOfDataPoints[row], + NumberOfPeaks: _numberOfPeaks[row], + BasePeakMz: _basePeakMz[row], + BasePeakIntensity: _basePeakIntensity[row], + TotalIonCurrent: _totalIonCurrent[row], + LowestObservedMz: _lowestObservedMz[row], + HighestObservedMz: _highestObservedMz[row], + DataProcessingRef: _spectrumDataProcessingRef[row], + NumberOfAuxiliaryArrays: _numberOfAuxiliaryArrays[row], + MzDeltaModel: mzDeltaModel, + ScanCombinationCurie: _scanCombinationCurie[row], + ParamGroupRefs: paramGroupRefs, + AuxArrays: AuxiliaryArrays.Parse(_spectrumAuxArraysJson[row]), + ValueArrayCurie: _valueArrayType[row], + ValueArrayUnitCurie: _valueArrayUnit[row]); + } + + /// + /// Returns the canonical mz/intensity arrays for a spectrum. Falls back to the + /// supplementary peaks layer only when the data layer has no rows for this + /// spectrum (shouldn't happen for a valid file). Whether the data is profile + /// or centroid is told by the spectrum description's CV classification, not + /// by which file the data lived in. + /// + public BinaryArrays? GetSpectrumData(int spectrumIndex) + { + if ((uint)spectrumIndex >= (uint)SpectrumCount) + throw new ArgumentOutOfRangeException(nameof(spectrumIndex)); + if (spectrumIndex >= _msSpectrumCount) + return _wavelength!.GetData(spectrumIndex - _msSpectrumCount); + + // The point layer keys on the spectrum's stored spectrum.index, which equals the logical + // position for pwiz-written files but need not for cross-stack / filtered lists — so look up + // by the stored index, not the row position. + long key = SpectrumDataKey(spectrumIndex); + var model = GetSpacingModel(spectrumIndex); + if (_dataLayer.Get(key, model) is { } d) return new BinaryArrays(d.Value, d.Intensity); + if (_peaksLayer.Get(key, model) is { } p) return new BinaryArrays(p.Value, p.Intensity); + return null; + } + + private long SpectrumDataKey(int spectrumIndex) => + (long)(_index[_primaryRowOfSpectrum[spectrumIndex]] ?? (ulong)spectrumIndex); + + /// The spectrum's mz_delta_model polynomial coefficients (for chunked-layout seam filling), + /// or null when the spectrum has no model. Forces the metadata group load that holds the column. + private double[]? GetSpacingModel(int spectrumIndex) + { + int row = _primaryRowOfSpectrum[spectrumIndex]; + EnsureGroupLoaded(GroupOfRow(row)); + if (_mzDeltaModel[row] is not { } coefs) return null; + var dense = coefs.Where(c => c.HasValue).Select(c => c!.Value).ToArray(); + return dense.Length > 0 ? dense : null; + } + + /// + /// Returns the supplementary centroid-peak layer for a spectrum (typically only + /// populated for profile spectra that were peak-picked alongside the profile signal). + /// Returns null when no supplementary peaks were stored for this spectrum. + /// + public BinaryArrays? GetSupplementaryPeaks(int spectrumIndex) + { + if ((uint)spectrumIndex >= (uint)SpectrumCount) + throw new ArgumentOutOfRangeException(nameof(spectrumIndex)); + // Wavelength spectra have no supplementary peaks layer. + if (spectrumIndex >= _msSpectrumCount) return null; + if (_peaksLayer.Get(SpectrumDataKey(spectrumIndex), GetSpacingModel(spectrumIndex)) is { } p) + return new BinaryArrays(p.Value, p.Intensity); + return null; + } + + public void Dispose() + { + // Close the open parquet readers (and their archive sub-streams) before removing any temp dir. + _metaReader?.Dispose(); + _dataLayer?.Dispose(); + _peaksLayer?.Dispose(); + _chromData?.Dispose(); + _wavelength?.Dispose(); + _archive?.Dispose(); + if (_tempDir is not null) + try { Directory.Delete(_tempDir, recursive: true); } catch { /* best-effort */ } + } + + // ----------------------- helpers ----------------------- + + private static string ExtractZip(string archivePath) + { + var dir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"mzpeak-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + ZipFile.ExtractToDirectory(archivePath, dir); + return dir; + } + + private static readonly Regex ListItemTrailerRegex = new(@"\.list\.(item|element)$", RegexOptions.Compiled); + + private static Dictionary BuildColumnIndex(SchemaDescriptor schema) + { + var map = new Dictionary(StringComparer.Ordinal); + for (int i = 0; i < schema.NumColumns; i++) + { + var col = schema.Column(i); + var dotPath = col.Path.ToDotString(); + map[dotPath] = i; + + // List-group leaves carry a `.list.item` (ParquetSharp/Arrow) or `.list.element` (real + // mzPeak) decoration, mid-path for nested structs (e.g. parameters) and trailing for scalar + // lists (e.g. mz_delta_model). Different writers pick different spellings, so register both + // forms — our callers always look up the `.list.element` spelling. + if (dotPath.Contains(".list.item", StringComparison.Ordinal)) + { + var alt = dotPath.Replace(".list.item", ".list.element"); + if (!map.ContainsKey(alt)) map[alt] = i; + } + else if (dotPath.Contains(".list.element", StringComparison.Ordinal)) + { + var alt = dotPath.Replace(".list.element", ".list.item"); + if (!map.ContainsKey(alt)) map[alt] = i; + } + + // Also register the path with any trailing list-element decoration stripped, so callers + // that look up the bare path get the same column. + var stripped = ListItemTrailerRegex.Replace(dotPath, ""); + if (!map.ContainsKey(stripped)) map[stripped] = i; + + var match = CvColumnRegex.Match(dotPath); + if (match.Success) + { + var accession = $"MS:{match.Groups["acc"].Value}"; + if (!map.ContainsKey(accession)) + map[accession] = i; + } + } + return map; + } + + // Lenient column reads — Phase 6 needs the reader to tolerate writers that + // emit a subset of the spec's columns (e.g. our minimal Phase 6 writer + // skips nested list columns). Missing columns return empty arrays of the + // requested length so downstream code can treat absent == empty uniformly. + // A column that IS present but whose physical type none of the adaptive + // candidates can decode throws instead — silently returning empties there + // would masquerade as an absent column and hide real data loss. + private static T?[] ReadNullableValue(RowGroupReader rg, Dictionary indices, string key, int rowCount) where T : unmanaged + { + if (!indices.TryGetValue(key, out int col)) return new T?[rowCount]; + + // Fast path: the column's element type matches T exactly (always true for pwiz-written files). + if (TryRead(rg, col, rowCount, v => v, out var exact)) return exact; + + // Adaptive path: another stack (e.g. mzPeak.NET) wrote a narrower/different physical type — + // double scalars as float32, int64 scalars as int32, etc. Read the actual type and convert. + if (typeof(T) == typeof(double)) + { + if (TryRead(rg, col, rowCount, v => (T)(object)(double)v, out var r)) return r; + } + else if (typeof(T) == typeof(long)) + { + if (TryRead(rg, col, rowCount, v => (T)(object)(long)v, out var r)) return r; + // mzPeak.NET writes the count fields (number_of_data_points / number_of_peaks) as unsigned. + if (TryRead(rg, col, rowCount, v => (T)(object)(long)v, out var r2)) return r2; + if (TryRead(rg, col, rowCount, v => (T)(object)(long)v, out var r3)) return r3; + } + else if (typeof(T) == typeof(ulong)) + { + if (TryRead(rg, col, rowCount, v => (T)(object)(ulong)v, out var r)) return r; + if (TryRead(rg, col, rowCount, v => (T)(object)(ulong)v, out var r2)) return r2; + if (TryRead(rg, col, rowCount, v => (T)(object)(ulong)v, out var r3)) return r3; + } + else if (typeof(T) == typeof(uint)) + { + if (TryRead(rg, col, rowCount, v => (T)(object)(uint)v, out var r)) return r; + if (TryRead(rg, col, rowCount, v => (T)(object)(uint)v, out var r2)) return r2; + } + throw UnsupportedColumnType(rg, col, key, typeof(T)); + } + + /// Read a column as nullable and convert each value to T; false on type mismatch. + private static bool TryRead(RowGroupReader rg, int col, int rowCount, Func conv, out T?[] result) + where TNat : unmanaged where T : unmanaged + { + try + { + using var cr = rg.Column(col).LogicalReader(); + var buf = new TNat?[rowCount]; + cr.ReadBatch(buf.AsSpan()); + var outBuf = new T?[rowCount]; + for (int i = 0; i < rowCount; i++) + if (buf[i] is TNat v) outBuf[i] = conv(v); + result = outBuf; + return true; + } + catch (InvalidCastException) + { + result = null!; + return false; + } + } + + /// Error for a column that exists but whose physical type none of the adaptive read + /// candidates could decode. Failing loudly here (rather than returning empty/NaN arrays that + /// look like an absent column) surfaces an unsupported file instead of silently losing data. + private static Exception UnsupportedColumnType(RowGroupReader rg, int col, string label, Type requested) + { + string physical; + try { using var cc = rg.MetaData.GetColumnChunkMetaData(col); physical = cc.Type.ToString(); } + catch { physical = "unknown"; } + return new NotSupportedException( + $"mzPeak column '{label}' has physical type {physical}, which the reader cannot convert to " + + $"{requested.Name}. The file may use a type or encoding from another writer that this reader does not support."); + } + + private static string?[] ReadNullableString(RowGroupReader rg, Dictionary indices, string key, int rowCount) + { + if (!indices.TryGetValue(key, out int col)) return new string?[rowCount]; + using var cr = rg.Column(col).LogicalReader(); + var buf = new string?[rowCount]; + cr.ReadBatch(buf.AsSpan()); + return buf; + } + + private static int?[] ReadNullableInt8(RowGroupReader rg, Dictionary indices, string key, int rowCount) + { + if (!indices.TryGetValue(key, out int col)) return new int?[rowCount]; + // 8-bit columns may be written signed (our writer, needed for scan_polarity ±1) or unsigned + // (mzPeak.NET writes ms_level as uint8). Widen either to int via the shared adaptive reader. + if (TryRead(rg, col, rowCount, v => v, out var r)) return r; + if (TryRead(rg, col, rowCount, v => v, out var r2)) return r2; + throw UnsupportedColumnType(rg, col, key, typeof(int)); + } + + /// Reads a small-integer column as nullable int, tolerating the actual physical width/signedness. + private static int?[] ReadNullableValueAsInt(RowGroupReader rg, Dictionary indices, string key, int rowCount) + { + if (!indices.TryGetValue(key, out int col)) return new int?[rowCount]; + // Fast path: our writer uses Int32. Other stacks may write UInt32 (mzPeak.NET's + // number_of_auxiliary_arrays) or narrower widths — read the actual type and widen. + if (TryRead(rg, col, rowCount, v => v, out var r)) return r; + if (TryRead(rg, col, rowCount, v => (int)v, out var r2)) return r2; + if (TryRead(rg, col, rowCount, v => v, out var r3)) return r3; + if (TryRead(rg, col, rowCount, v => v, out var r4)) return r4; + throw UnsupportedColumnType(rg, col, key, typeof(int)); + } + + private static string?[]?[] ReadNullableStringList(RowGroupReader rg, Dictionary indices, string key, int rowCount) + { + if (!indices.TryGetValue(key, out int col)) return new string?[rowCount][]; + using var cr = rg.Column(col).LogicalReader(); + var buf = new string?[rowCount][]; + cr.ReadBatch(buf.AsSpan()); + return buf; + } + + private static T?[]?[] ReadNullableValueList(RowGroupReader rg, Dictionary indices, string key, int rowCount) where T : unmanaged + { + if (!indices.TryGetValue(key, out int col)) return new T?[rowCount][]; + try + { + using var cr = rg.Column(col).LogicalReader(); + var buf = new T?[rowCount][]; + cr.ReadBatch(buf.AsSpan()); + return buf; + } + catch (InvalidCastException) when (typeof(T) == typeof(double)) + { + // float32 list column written by another stack; widen each element to double. + using var cr = rg.Column(col).LogicalReader(); + var fbuf = new float?[rowCount][]; + cr.ReadBatch(fbuf.AsSpan()); + var buf = new T?[rowCount][]; + for (int i = 0; i < rowCount; i++) + { + if (fbuf[i] is not { } fa) continue; + var outArr = new T?[fa.Length]; + for (int j = 0; j < fa.Length; j++) + if (fa[j] is float f) outArr[j] = (T)(object)(double)f; + buf[i] = outArr; + } + return buf; + } + } + + /// + /// Reconstruct a list of by zipping the seven parallel leaf-column + /// arrays for one row. The shortest non-null array's length wins (defensive against + /// writer-side parallel-array length drift); in practice all arrays are the same length + /// per the mzPeak spec. + /// + private static IReadOnlyList ZipParams( + string?[]? name, string?[]? accession, + string?[]? valueString, long?[]? valueInteger, + double?[]? valueFloat, bool?[]? valueBoolean, + string?[]? unit, string?[]? type = null) + { + int n = MinNonNullLength(name, accession, valueString, valueInteger, valueFloat, valueBoolean, unit); + if (n == 0) return Array.Empty(); + var list = new CvParam[n]; + for (int i = 0; i < n; i++) + { + list[i] = new CvParam( + Name: name?[i], Accession: accession?[i], + ValueString: valueString?[i], ValueInteger: valueInteger?[i], + ValueFloat: valueFloat?[i], ValueBoolean: valueBoolean?[i], + Unit: unit?[i], Type: type?[i]); + } + return list; + } + + private static IReadOnlyList ZipScanWindows(double?[]? lower, double?[]? upper) + { + int n = MinNonNullLength(lower, upper); + if (n == 0) return Array.Empty(); + var list = new ScanWindow[n]; + for (int i = 0; i < n; i++) + list[i] = new ScanWindow(lower?[i], upper?[i]); + return list; + } + + private static int MinNonNullLength(params Array?[] arrays) + { + int min = int.MaxValue; + bool any = false; + foreach (var a in arrays) + { + if (a is null) continue; + any = true; + if (a.Length < min) min = a.Length; + } + return any ? min : 0; + } + + /// + /// UV/DAD wavelength spectra read from the dedicated wavelength_spectra_metadata.parquet / + /// wavelength_spectra_data.parquet entries some writers (mzPeak.NET) use instead of inlining + /// them in spectra_metadata. The schema mirrors a normal spectrum table except the value + /// array is wavelength (nm) rather than m/z, and there are no ms_level/polarity/precursor columns. + /// These tables are secondary and typically far smaller than the MS table, so columns are read + /// eagerly at open; binary data still flows through a lazy point layout. Logical spectrum indices + /// at or above the MS spectrum count route here (index − msCount → local row). + /// + private sealed class WavelengthSpectra : IDisposable + { + private const string WavelengthArrayCurie = "MS:1000617"; // wavelength array + private const string NanometerCurie = "UO:0000018"; // nanometer + + public int Count { get; } + + private readonly ulong?[] _index; + private readonly string?[] _id, _representationCurie, _typeCurie, _filterString, _dpRef, _auxJson, _scanIonMobType; + private readonly double?[] _time, _scanStartTime, _ionInjTime, _ionMobValue, + _lowestWl, _highestWl, _lambdaMax, _basePeakIntensity, _tic; + private readonly long?[] _numDataPoints, _presetScanConfig; + private readonly uint?[] _instrConfigRef; + private readonly int?[] _numAuxArrays; + private readonly ulong?[] _scanSourceIndex; + + private readonly string?[]?[] _spName, _spAcc, _spValStr, _spUnit, _spType; + private readonly long?[]?[] _spValInt; + private readonly double?[]?[] _spValFloat; + private readonly bool?[]?[] _spValBool; + + private readonly string?[]?[] _scName, _scAcc, _scValStr, _scUnit, _scType; + private readonly long?[]?[] _scValInt; + private readonly double?[]?[] _scValFloat; + private readonly bool?[]?[] _scValBool; + + private readonly IPointLayout _data; + + public WavelengthSpectra(ParquetFileReader meta, ParquetFileReader? data) + { + using (meta) + { + var cols = BuildColumnIndex(meta.FileMetaData.Schema); + int groups = meta.FileMetaData.NumRowGroups; + var offs = new int[groups + 1]; + for (int g = 0; g < groups; g++) + { + using var rg = meta.RowGroup(g); + offs[g + 1] = offs[g] + checked((int)rg.MetaData.NumRows); + } + int n = offs[groups]; + Count = n; + + _index = new ulong?[n]; _id = new string?[n]; _representationCurie = new string?[n]; + _typeCurie = new string?[n]; _filterString = new string?[n]; _dpRef = new string?[n]; + _auxJson = new string?[n]; _scanIonMobType = new string?[n]; + _time = new double?[n]; _scanStartTime = new double?[n]; _ionInjTime = new double?[n]; + _ionMobValue = new double?[n]; _lowestWl = new double?[n]; _highestWl = new double?[n]; + _lambdaMax = new double?[n]; _basePeakIntensity = new double?[n]; _tic = new double?[n]; + _numDataPoints = new long?[n]; _presetScanConfig = new long?[n]; + _instrConfigRef = new uint?[n]; _numAuxArrays = new int?[n]; _scanSourceIndex = new ulong?[n]; + _spName = new string?[n][]; _spAcc = new string?[n][]; _spValStr = new string?[n][]; + _spUnit = new string?[n][]; _spType = new string?[n][]; _spValInt = new long?[n][]; + _spValFloat = new double?[n][]; _spValBool = new bool?[n][]; + _scName = new string?[n][]; _scAcc = new string?[n][]; _scValStr = new string?[n][]; + _scUnit = new string?[n][]; _scType = new string?[n][]; _scValInt = new long?[n][]; + _scValFloat = new double?[n][]; _scValBool = new bool?[n][]; + + for (int g = 0; g < groups; g++) + { + using var rg = meta.RowGroup(g); + int len = offs[g + 1] - offs[g], o = offs[g]; + CopyInto(_index, ReadNullableValue(rg, cols, "spectrum.index", len), o); + CopyInto(_id, ReadNullableString(rg, cols, "spectrum.id", len), o); + CopyInto(_time, ReadNullableValue(rg, cols, "spectrum.time", len), o); + CopyInto(_representationCurie, ReadNullableString(rg, cols, "MS:1000525", len), o); + CopyInto(_typeCurie, ReadNullableString(rg, cols, "MS:1000559", len), o); + CopyInto(_lowestWl, ReadNullableValue(rg, cols, "MS:1000619", len), o); + CopyInto(_highestWl, ReadNullableValue(rg, cols, "MS:1000618", len), o); + CopyInto(_lambdaMax, ReadNullableValue(rg, cols, "MS:1003812", len), o); + CopyInto(_basePeakIntensity, ReadNullableValue(rg, cols, "MS:1000505", len), o); + CopyInto(_tic, ReadNullableValue(rg, cols, "MS:1000285", len), o); + CopyInto(_numDataPoints, ReadNullableValue(rg, cols, "MS:1003060", len), o); + CopyInto(_dpRef, ReadNullableString(rg, cols, "spectrum.data_processing_ref", len), o); + CopyInto(_numAuxArrays, ReadNullableValueAsInt(rg, cols, "spectrum.number_of_auxiliary_arrays", len), o); + CopyInto(_auxJson, ReadNullableString(rg, cols, "spectrum.auxiliary_arrays", len), o); + + CopyInto(_scanSourceIndex, ReadNullableValue(rg, cols, "scan.source_index", len), o); + CopyInto(_scanStartTime, ReadNullableValue(rg, cols, "MS:1000016", len), o); + CopyInto(_filterString, ReadNullableString(rg, cols, "MS:1000512", len), o); + CopyInto(_ionInjTime, ReadNullableValue(rg, cols, "MS:1000927", len), o); + CopyInto(_presetScanConfig, ReadNullableValue(rg, cols, "MS:1000616", len), o); + CopyInto(_instrConfigRef, ReadNullableValue(rg, cols, "scan.instrument_configuration_ref", len), o); + CopyInto(_ionMobValue, ReadNullableValue(rg, cols, "scan.ion_mobility_value", len), o); + CopyInto(_scanIonMobType, ReadNullableString(rg, cols, "scan.ion_mobility_type", len), o); + + CopyInto(_spName, ReadNullableStringList(rg, cols, "spectrum.parameters.list.element.name", len), o); + CopyInto(_spAcc, ReadNullableStringList(rg, cols, "spectrum.parameters.list.element.accession", len), o); + CopyInto(_spValStr, ReadNullableStringList(rg, cols, "spectrum.parameters.list.element.value.string", len), o); + CopyInto(_spValInt, ReadNullableValueList(rg, cols, "spectrum.parameters.list.element.value.integer", len), o); + CopyInto(_spValFloat, ReadNullableValueList(rg, cols, "spectrum.parameters.list.element.value.float", len), o); + CopyInto(_spValBool, ReadNullableValueList(rg, cols, "spectrum.parameters.list.element.value.boolean", len), o); + CopyInto(_spUnit, ReadNullableStringList(rg, cols, "spectrum.parameters.list.element.unit", len), o); + CopyInto(_spType, ReadNullableStringList(rg, cols, "spectrum.parameters.list.element.type", len), o); + + CopyInto(_scName, ReadNullableStringList(rg, cols, "scan.parameters.list.element.name", len), o); + CopyInto(_scAcc, ReadNullableStringList(rg, cols, "scan.parameters.list.element.accession", len), o); + CopyInto(_scValStr, ReadNullableStringList(rg, cols, "scan.parameters.list.element.value.string", len), o); + CopyInto(_scValInt, ReadNullableValueList(rg, cols, "scan.parameters.list.element.value.integer", len), o); + CopyInto(_scValFloat, ReadNullableValueList(rg, cols, "scan.parameters.list.element.value.float", len), o); + CopyInto(_scValBool, ReadNullableValueList(rg, cols, "scan.parameters.list.element.value.boolean", len), o); + CopyInto(_scUnit, ReadNullableStringList(rg, cols, "scan.parameters.list.element.unit", len), o); + CopyInto(_scType, ReadNullableStringList(rg, cols, "scan.parameters.list.element.type", len), o); + } + } + _data = MakePointLayout(data); + } + + public string GetId(int i) => _id[i] ?? string.Empty; + + public BinaryArrays? GetData(int i) + { + long key = (long)(_index[i] ?? (ulong)i); + return _data.Get(key) is { } d ? new BinaryArrays(d.Value, d.Intensity) : null; + } + + public SpectrumDescription GetDescription(int i) + { + var parameters = new List(ZipParams( + _spName[i], _spAcc[i], _spValStr[i], _spValInt[i], _spValFloat[i], _spValBool[i], _spUnit[i], _spType[i])); + + // The typed wavelength scalars carry their own CV terms (distinct from the m/z ones), so + // surface them as params for the translation layer rather than reusing the m/z accessors. + if (!string.IsNullOrEmpty(_typeCurie[i])) + parameters.Add(new CvParam("spectrum type", _typeCurie[i], "", null, null, null, null)); + AddScalar(parameters, "MS:1000619", "lowest observed wavelength", _lowestWl[i], NanometerCurie); + AddScalar(parameters, "MS:1000618", "highest observed wavelength", _highestWl[i], NanometerCurie); + AddScalar(parameters, "MS:1003812", "lambda max", _lambdaMax[i], NanometerCurie); + AddScalar(parameters, "MS:1000505", "base peak intensity", _basePeakIntensity[i], "MS:1000131"); + AddScalar(parameters, "MS:1000285", "total ion current", _tic[i], null); + + var scanParams = ZipParams( + _scName[i], _scAcc[i], _scValStr[i], _scValInt[i], _scValFloat[i], _scValBool[i], _scUnit[i], _scType[i]); + + ScanInfo? scan = (_scanStartTime[i] is null && _filterString[i] is null && _instrConfigRef[i] is null + && _ionInjTime[i] is null && _scanSourceIndex[i] is null && scanParams.Count == 0) + ? null + : new ScanInfo( + StartTime: _scanStartTime[i], + FilterString: _filterString[i], + InstrumentConfigurationRef: _instrConfigRef[i], + IonInjectionTime: _ionInjTime[i], + Parameters: scanParams, + ScanWindows: Array.Empty(), + SourceIndex: _scanSourceIndex[i], + IonMobilityValue: _ionMobValue[i], + IonMobilityTypeCurie: _scanIonMobType[i], + PresetScanConfiguration: _presetScanConfig[i]); + + var curie = _representationCurie[i]; + return new SpectrumDescription( + Index: _index[i] ?? (ulong)i, + Id: _id[i] ?? string.Empty, + Time: _time[i] ?? 0.0, + MsLevel: null, + IsProfile: curie == CurieProfileSpectrum, + IsCentroid: curie == CurieCentroidSpectrum, + RepresentationCurie: curie, + Scan: scan, + Precursors: Array.Empty(), + Parameters: parameters, + SpectrumTypeCurie: _typeCurie[i], + NumberOfDataPoints: _numDataPoints[i], + DataProcessingRef: _dpRef[i], + NumberOfAuxiliaryArrays: _numAuxArrays[i], + AuxArrays: AuxiliaryArrays.Parse(_auxJson[i]), + ValueArrayCurie: WavelengthArrayCurie, + ValueArrayUnitCurie: NanometerCurie); + } + + private static void AddScalar(List list, string accession, string name, double? value, string? unit) + { + if (value is double d) + list.Add(new CvParam(name, accession, null, null, d, null, unit)); + } + + public void Dispose() => _data.Dispose(); + } + + /// + /// Lazily reads a point-layout parquet — (parent_index, value, intensity) — one row group at a + /// time, caching the most-recently-touched groups (LRU). The writer splits row groups on + /// parent (spectrum/chromatogram) boundaries and records each group's parent-index range in the + /// point_row_group_ranges KV, so reads only the single group covering + /// the requested parent. Open is cheap (no point data read) and memory is bounded to a few + /// row groups instead of the whole file. Reads are guarded by a lock so concurrent callers are + /// safe; the underlying parquet reader stays open until . + /// + private sealed class LazyPointLayout : IPointLayout + { + // Must match MzPeakWriter.PointRowGroupRangesKey. + private const string PointRowGroupRangesKey = "point_row_group_ranges"; + private const int MaxCachedGroups = 3; + private readonly ParquetFileReader? _reader; + private readonly (long First, long Last)[] _ranges; + private readonly int _colIdx, _colVal, _colInt; + private readonly object _lock = new(); + private readonly Dictionary> _cache = new(); + private readonly LinkedList _lru = new(); + + public LazyPointLayout(ParquetFileReader? reader) + { + _ranges = Array.Empty<(long, long)>(); + _colIdx = _colVal = _colInt = -1; + if (reader is null) return; + + _reader = reader; + var schema = _reader.FileMetaData.Schema; + for (int i = 0; i < schema.NumColumns; i++) + { + var p = schema.Column(i).Path.ToDotString(); + if (p.EndsWith(".intensity", StringComparison.Ordinal)) _colInt = i; + else if (p.EndsWith("_index", StringComparison.Ordinal)) _colIdx = i; + else if (p.EndsWith(".mz", StringComparison.Ordinal) || p.EndsWith(".time", StringComparison.Ordinal) + || p.EndsWith(".wavelength", StringComparison.Ordinal)) _colVal = i; + } + + var kv = _reader.FileMetaData.KeyValueMetadata; + if (kv.TryGetValue(PointRowGroupRangesKey, out var json)) + { + // pwiz-written files: the writer splits row groups on parent-index boundaries and + // records the authoritative [first,last] per group, so groups never overlap. + var raw = System.Text.Json.JsonSerializer.Deserialize(json); + if (raw is not null) _ranges = raw.Select(r => (r[0], r[1])).ToArray(); + } + else if (_colIdx >= 0) + { + // Files without the range KV (single-row-group pwiz files, and cross-stack files such + // as mzPeak.NET) derive each row group's parent-index [min,max] range. These can + // OVERLAP (a foreign writer may split a parent across groups), so Get() merges every + // group whose range covers the key rather than taking the first. + _ranges = DeriveRangesFromColumn(_reader, _colIdx); + } + } + + public (double[] Value, float[] Intensity)? Get(long parentIndex, double[]? spacingModel = null) + { + if (_reader is null || _colIdx < 0 || _colVal < 0 || _colInt < 0) return null; + + // Collect every row group whose [min,max] covers the key. With the KV path these ranges + // are disjoint partitions so this is exactly one group; with derived (possibly overlapping) + // ranges a spectrum's points may live in several, and we concatenate them in group order. + double[]? value = null; + float[]? intensity = null; + for (int i = 0; i < _ranges.Length; i++) + { + if (parentIndex < _ranges[i].First || parentIndex > _ranges[i].Last) continue; + lock (_lock) + { + var group = EnsureLoaded(i); + if (!group.TryGetValue(parentIndex, out var v)) continue; + if (value is null) { value = v.Value; intensity = v.Intensity; } + else { value = Concat(value, v.Value); intensity = Concat(intensity!, v.Intensity); } + } + } + if (value is null) return null; + + // mzPeak.NET stores the value axis with null gaps (the NullInterpolate transform) to be + // reconstructed from the spectrum's spacing model; null reads back as NaN here. Fill those + // from the model exactly as the chunked layout does, so both encodings agree. + if (spacingModel is not null && Array.Exists(value, double.IsNaN)) + { + var nullable = new double?[value.Length]; + for (int i = 0; i < value.Length; i++) + nullable[i] = double.IsNaN(value[i]) ? (double?)null : value[i]; + value = MzPeakChunkCodec.FillNullsWithModel(nullable, spacingModel); + } + return (value, intensity!); + } + + private static T[] Concat(T[] a, T[] b) + { + var r = new T[a.Length + b.Length]; + Array.Copy(a, 0, r, 0, a.Length); + Array.Copy(b, 0, r, a.Length, b.Length); + return r; + } + + private Dictionary EnsureLoaded(int rgIdx) + { + if (_cache.TryGetValue(rgIdx, out var cached)) + { + _lru.Remove(rgIdx); + _lru.AddFirst(rgIdx); + return cached; + } + var group = LoadGroup(rgIdx); + _cache[rgIdx] = group; + _lru.AddFirst(rgIdx); + while (_lru.Count > MaxCachedGroups) + { + int evict = _lru.Last!.Value; + _lru.RemoveLast(); + _cache.Remove(evict); + } + return group; + } + + private Dictionary LoadGroup(int rgIdx) + { + using var rg = _reader!.RowGroup(rgIdx); + int rowCount = checked((int)rg.MetaData.NumRows); + // Adaptive reads: cross-stack writers vary the physical type of these columns (mzPeak.NET + // stores the wavelength value array as float32; index widths differ too). + var idx = ReadIndexColumn(rg, _colIdx, rowCount); + var val = ReadDoubleColumn(rg, _colVal, rowCount); + var inten = ReadFloatColumn(rg, _colInt, rowCount); + + var counts = new Dictionary(); + for (int r = 0; r < rowCount; r++) + if (idx[r] is { } u) counts[(long)u] = counts.GetValueOrDefault((long)u) + 1; + + var outv = new Dictionary(counts.Count); + var cursor = new Dictionary(counts.Count); + foreach (var kvp in counts) + { + outv[kvp.Key] = (new double[kvp.Value], new float[kvp.Value]); + cursor[kvp.Key] = 0; + } + for (int r = 0; r < rowCount; r++) + { + if (idx[r] is not { } u) continue; + long s = (long)u; + var (vv, ii) = outv[s]; + int c = cursor[s]++; + // A null value-axis point is a gap to be interpolated from the spacing model (done in + // Get); mark it NaN until then. A null intensity is zero (the NullZero transform). + vv[c] = val[r] ?? double.NaN; + ii[c] = inten[r] ?? 0f; + } + return outv; + } + + // Column reads tolerant of the physical type a foreign stack chose. Each tries the canonical + // type first (pwiz's own files) then falls back to the other plausible width/precision, reusing + // the enclosing reader's adaptive primitive. A present column with no matching candidate throws + // (rather than yielding NaN points that look like real data). + private static ulong?[] ReadIndexColumn(RowGroupReader rg, int col, int n) + { + if (TryRead(rg, col, n, v => v, out var r)) return r; + if (TryRead(rg, col, n, v => (ulong)v, out var r2)) return r2; + if (TryRead(rg, col, n, v => v, out var r3)) return r3; + if (TryRead(rg, col, n, v => (ulong)v, out var r4)) return r4; + throw UnsupportedColumnType(rg, col, "point index", typeof(ulong)); + } + + private static double?[] ReadDoubleColumn(RowGroupReader rg, int col, int n) + { + if (TryRead(rg, col, n, v => v, out var r)) return r; + if (TryRead(rg, col, n, v => v, out var r2)) return r2; + throw UnsupportedColumnType(rg, col, "point value", typeof(double)); + } + + private static float?[] ReadFloatColumn(RowGroupReader rg, int col, int n) + { + if (TryRead(rg, col, n, v => v, out var r)) return r; + if (TryRead(rg, col, n, v => (float)v, out var r2)) return r2; + throw UnsupportedColumnType(rg, col, "point intensity", typeof(float)); + } + + public void Dispose() => _reader?.Dispose(); + } + + /// + /// Reads the mzPeak "chunked" point layout (mzPeak.NET's compressed variant): one parquet row per + /// m/z chunk — spectrum_index, mz_chunk_start, a delta/no-compression encoded + /// mz_chunk_values list, a chunk_encoding CURIE, and a parallel intensity list. + /// Like it reads one row group at a time with an LRU cache; per group + /// it buckets the raw chunks by spectrum index, decoding+concatenating them (and filling any + /// delta-seam nulls from the per-spectrum spacing model) on demand in . + /// + private sealed class ChunkedPointLayout : IPointLayout + { + private const int MaxCachedGroups = 3; + + private readonly record struct Chunk(double Start, double?[] Mz, string? Encoding, float?[] Intensity); + + private readonly ParquetFileReader _reader; + private readonly (long First, long Last)[] _ranges; + private readonly int _colIdx, _colStart, _colValues, _colEncoding, _colIntensity; + private readonly object _lock = new(); + private readonly Dictionary>> _cache = new(); + private readonly LinkedList _lru = new(); + + /// True when the data parquet uses the chunked layout (it carries a chunk_encoding column). + public static bool IsChunked(ParquetFileReader reader) + { + var schema = reader.FileMetaData.Schema; + for (int i = 0; i < schema.NumColumns; i++) + if (schema.Column(i).Path.ToDotString().Contains("chunk_encoding", StringComparison.Ordinal)) + return true; + return false; + } + + public ChunkedPointLayout(ParquetFileReader reader) + { + _reader = reader; + _colIdx = _colStart = _colValues = _colEncoding = _colIntensity = -1; + var schema = _reader.FileMetaData.Schema; + for (int i = 0; i < schema.NumColumns; i++) + { + var p = schema.Column(i).Path.ToDotString(); + if (p.Contains("spectrum_index", StringComparison.Ordinal)) _colIdx = i; + else if (p.Contains("chunk_start", StringComparison.Ordinal)) _colStart = i; + else if (p.Contains("chunk_encoding", StringComparison.Ordinal)) _colEncoding = i; + else if (p.Contains("chunk_values", StringComparison.Ordinal)) _colValues = i; + else if (p.Contains("intensity", StringComparison.Ordinal)) _colIntensity = i; + } + _ranges = _colIdx >= 0 ? DeriveRangesFromColumn(_reader, _colIdx) : Array.Empty<(long, long)>(); + } + + public (double[] Value, float[] Intensity)? Get(long parentIndex, double[]? spacingModel = null) + { + if (_colIdx < 0 || _colStart < 0 || _colValues < 0 || _colEncoding < 0 || _colIntensity < 0) return null; + + var mz = new List(); + var intensity = new List(); + bool found = false; + for (int i = 0; i < _ranges.Length; i++) + { + if (parentIndex < _ranges[i].First || parentIndex > _ranges[i].Last) continue; + lock (_lock) + { + var group = EnsureLoaded(i); + if (!group.TryGetValue(parentIndex, out var chunks)) continue; + found = true; + foreach (var chunk in chunks) + { + var decoded = MzPeakChunkCodec.DecodeMz(chunk.Encoding, chunk.Start, chunk.Mz); + bool hasNull = false; + foreach (var v in decoded) if (v is null) { hasNull = true; break; } + mz.AddRange(hasNull && spacingModel is not null + ? MzPeakChunkCodec.FillNullsWithModel(decoded, spacingModel) + : MzPeakChunkCodec.ToDense(decoded)); + intensity.AddRange(MzPeakChunkCodec.IntensityToDense(chunk.Intensity)); + } + } + } + return found ? (mz.ToArray(), intensity.ToArray()) : null; + } + + private Dictionary> EnsureLoaded(int rgIdx) + { + if (_cache.TryGetValue(rgIdx, out var cached)) + { + _lru.Remove(rgIdx); + _lru.AddFirst(rgIdx); + return cached; + } + var group = LoadGroup(rgIdx); + _cache[rgIdx] = group; + _lru.AddFirst(rgIdx); + while (_lru.Count > MaxCachedGroups) + { + int evict = _lru.Last!.Value; + _lru.RemoveLast(); + _cache.Remove(evict); + } + return group; + } + + private Dictionary> LoadGroup(int rgIdx) + { + using var rg = _reader.RowGroup(rgIdx); + int n = checked((int)rg.MetaData.NumRows); + + if (!TryRead(rg, _colIdx, n, v => v, out var sidx) + && !TryRead(rg, _colIdx, n, v => (ulong)v, out sidx)) + throw UnsupportedColumnType(rg, _colIdx, "chunk spectrum_index", typeof(ulong)); + + var start = new double?[n]; + using (var c = rg.Column(_colStart).LogicalReader()) c.ReadBatch(start.AsSpan()); + var encoding = new string?[n]; + using (var c = rg.Column(_colEncoding).LogicalReader()) c.ReadBatch(encoding.AsSpan()); + var values = new double?[n][]; + using (var c = rg.Column(_colValues).LogicalReader()) c.ReadBatch(values.AsSpan()); + var intensity = new float?[n][]; + using (var c = rg.Column(_colIntensity).LogicalReader()) c.ReadBatch(intensity.AsSpan()); + + var bucket = new Dictionary>(); + for (int r = 0; r < n; r++) + { + // A null start marks an unused row (mzPeak.NET skips it). + if (sidx[r] is not { } s || start[r] is not { } st) continue; + if (!bucket.TryGetValue((long)s, out var list)) bucket[(long)s] = list = new List(); + list.Add(new Chunk(st, values[r] ?? Array.Empty(), encoding[r], intensity[r] ?? Array.Empty())); + } + return bucket; + } + + public void Dispose() => _reader.Dispose(); + } +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakWriter.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakWriter.cs new file mode 100644 index 00000000000..9264ef99648 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/MzPeakWriter.cs @@ -0,0 +1,948 @@ +using System.IO.Compression; +using System.Text.Json; +using ParquetSharp; +using ParquetSharp.Schema; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// Phase 6+ writer using ParquetSharp's GroupNode schema API. Critical for +/// cross-stack compatibility: ParquetSharp's `Column<T>("foo.bar")` shortcut +/// produces a flat leaf with a literal-dot name (so Arrow sees one column called +/// "foo.bar"), but the mzPeak spec — and every other mzPeak consumer +/// (mzPeak.NET, pyarrow, Rust mzdata) — expects nested group structures, so +/// Arrow sees one Struct column called "foo" containing a field "bar". +/// +/// This writer builds explicit GroupNode trees per file: each top-level entity +/// (spectrum, scan, precursor, selected_ion, chromatogram, point) is an Optional +/// GroupNode, with its CV-encoded leaves and parameter lists nested inside. +/// List columns use the 3-level Apache Arrow encoding (optional outer LIST → +/// repeated `list` group → optional `element`). +/// +public sealed class MzPeakWriter +{ + /// + /// One precursor of a spectrum. Multi-precursor spectra carry several of + /// these; mzPeak.NET reads precursors as a separate row-set keyed by + /// source_index (the parent spectrum's row index), with + /// precursor_index distinguishing siblings. The writer fans each + /// spectrum into max(1, Precursors.Count) metadata rows so every + /// precursor lands in its own row. + /// + public sealed record PrecursorToWrite( + string? PrecursorId = null, + double? IsolationTargetMz = null, double? IsolationLowerOffset = null, double? IsolationUpperOffset = null, + double? CollisionEnergy = null, string? DissociationMethodCurie = null, + // One selected ion per precursor (mzPeak supports lists; we model the common 1:1 case). + double? SelectedIonMz = null, double? SelectedIonPeakIntensity = null, long? SelectedIonChargeState = null, + double? SelectedIonIonMobilityValue = null, string? SelectedIonIonMobilityTypeCurie = null, + IReadOnlyList? IsolationWindowParameters = null, + IReadOnlyList? ActivationParameters = null, + IReadOnlyList? SelectedIonParameters = null); + + public sealed record SpectrumToWrite( + ulong Index, string Id, double Time, int? MsLevel, bool IsProfile, + double[] Mz, float[] Intensity, + // Scan scalars + double? ScanStartTime = null, string? FilterString = null, + uint? InstrumentConfigurationRef = null, double? IonInjectionTime = null, + // scan[0] spectrumRef (source spectrum for a combined scan), if any. + string? ScanSpectrumRef = null, + // Lists + double?[]? ScanWindowLowerLimits = null, double?[]? ScanWindowUpperLimits = null, + // JSON of per-scan-window free-form params beyond the lower/upper limit columns. + string? ScanWindowParamsJson = null, + // JSON of scanList scans beyond scan[0] (combined ion-mobility spectra: one per mobility bin). + string? ExtraScansJson = null, + IReadOnlyList? SpectrumParameters = null, + IReadOnlyList? ScanParameters = null, + // Supplementary peaks + double[]? SupplementaryPeaksMz = null, float[]? SupplementaryPeaksIntensity = null, + // Spectrum-level CV-encoded scalars + int? ScanPolarity = null, // MS:1000465 (Int8 — +1 / -1) + string? SpectrumTypeCurie = null, // MS:1000559 + long? NumberOfDataPoints = null, // MS:1003060 (defaults to Mz.Length) + long? NumberOfPeaks = null, // MS:1003059 (defaults to SupplementaryPeaksMz?.Length) + double? BasePeakMz = null, // MS:1000504 + double? BasePeakIntensity = null, // MS:1000505 + double? TotalIonCurrent = null, // MS:1000285 + double? LowestObservedMz = null, // MS:1000528 + double? HighestObservedMz = null, // MS:1000527 + string? SpectrumDataProcessingRef = null, + int? NumberOfAuxiliaryArrays = null, + double?[]? MzDeltaModel = null, + // Scan linking + IMS + ulong? ScanSourceIndex = null, // defaults to Index + double? ScanIonMobilityValue = null, + string? ScanIonMobilityTypeCurie = null, + long? PresetScanConfiguration = null, // MS:1000616 + // scanList combination method CURIE (MS:1000570 children, e.g. MS:1000795 no combination). + string? ScanCombinationCurie = null, + // referenceableParamGroup ids this spectrum references. + IReadOnlyList? ParamGroupRefs = null, + // False when the source spectrum carries NEITHER profile nor centroid representation CV + // (so the reader emits no representation term instead of defaulting to one). Defaults true. + bool HasRepresentation = true, + // Non-null when the spectrum's value array isn't m/z (e.g. UV/DAD wavelength array): the + // array-type CURIE + its unit CURIE, so the reader rebuilds the right value array. + string? ValueArrayCurie = null, + string? ValueArrayUnitCurie = null, + // JSON of auxiliary (non-m/z, non-intensity) binary/integer arrays on this spectrum. + string? AuxiliaryArraysJson = null, + // Zero or more precursors; writer fans these into separate rows. + IReadOnlyList? Precursors = null); + + /// + /// Internal fan-out row: one row per (spectrum, precursor). Spectrum-level + /// and scan-level columns are written only on rows + /// (the first row for each spectrum); extra precursor rows get null in + /// those groups but a populated precursor + selected_ion group. + /// + private sealed record Row(SpectrumToWrite Spectrum, PrecursorToWrite? Precursor, int PrecursorIdx, bool IsPrimary); + + public sealed record ChromatogramToWrite( + ulong Index, string Id, string? ChromatogramTypeCurie, string? DataProcessingRef, + double[] Time, float[] Intensity, + // Unit CURIE of the time array (e.g. UO:0000010 second / UO:0000031 minute). mzML lets this + // vary, and the point-layout time column can't carry it, so it travels as its own column. + string? TimeUnitCurie = null, + // Unit CURIE of the intensity array (counts / % / psi / µL·min⁻¹ / pascal — varies widely). + string? IntensityUnitCurie = null, + // Free-form chromatogram-level CV/user params not represented by a typed column (polarity, …). + IReadOnlyList? Parameters = null, + // JSON of auxiliary (non-time, non-intensity) binary/integer arrays (e.g. the "ms level" int array). + string? AuxiliaryArraysJson = null); + + private const string CurieProfileSpectrum = "MS:1000128"; + private const string CurieCentroidSpectrum = "MS:1000127"; + + public static void Write( + string outputPath, + IReadOnlyList spectra, + FileMetadata fileMetadata, + IReadOnlyList? chromatograms = null) + { + chromatograms ??= Array.Empty(); + var stagingDir = Path.Combine(Path.GetTempPath(), $"mzpeak-write-{Guid.NewGuid():N}"); + Directory.CreateDirectory(stagingDir); + try + { + WriteSpectraMetadata(Path.Combine(stagingDir, "spectra_metadata.parquet"), spectra, fileMetadata); + WriteSpectraData(Path.Combine(stagingDir, "spectra_data.parquet"), spectra); + + bool hasSupplementary = spectra.Any(s => s.SupplementaryPeaksMz is { Length: > 0 }); + if (hasSupplementary) + WriteSpectraPeaks(Path.Combine(stagingDir, "spectra_peaks.parquet"), spectra); + if (chromatograms.Count > 0) + { + WriteChromatogramsMetadata(Path.Combine(stagingDir, "chromatograms_metadata.parquet"), chromatograms); + WriteChromatogramsData(Path.Combine(stagingDir, "chromatograms_data.parquet"), chromatograms); + } + WriteManifest(Path.Combine(stagingDir, "mzpeak_index.json"), chromatograms.Count > 0, hasSupplementary); + + if (File.Exists(outputPath)) File.Delete(outputPath); + ZipFile.CreateFromDirectory(stagingDir, outputPath, CompressionLevel.NoCompression, includeBaseDirectory: false); + } + finally + { + try { Directory.Delete(stagingDir, recursive: true); } catch { /* best-effort */ } + } + } + + // ---------------- Schema builder helpers ---------------- + + private static PrimitiveNode StringLeaf(string name) => new(name, Repetition.Optional, LogicalType.String(), PhysicalType.ByteArray); + private static PrimitiveNode DoubleLeaf(string name) => new(name, Repetition.Optional, LogicalType.None(), PhysicalType.Double); + private static PrimitiveNode FloatLeaf(string name) => new(name, Repetition.Optional, LogicalType.None(), PhysicalType.Float); + private static PrimitiveNode BoolLeaf(string name) => new(name, Repetition.Optional, LogicalType.None(), PhysicalType.Boolean); + private static PrimitiveNode LongLeaf(string name) => new(name, Repetition.Optional, LogicalType.None(), PhysicalType.Int64); + private static PrimitiveNode UInt64Leaf(string name) => new(name, Repetition.Optional, LogicalType.Int(64, isSigned: false), PhysicalType.Int64); + private static PrimitiveNode UInt32Leaf(string name) => new(name, Repetition.Optional, LogicalType.Int(32, isSigned: false), PhysicalType.Int32); + private static PrimitiveNode Int8Leaf(string name) => new(name, Repetition.Optional, LogicalType.Int(8, isSigned: true), PhysicalType.Int32); + + private static GroupNode Struct(string name, params Node[] children) => new(name, Repetition.Optional, children); + + /// + /// 3-level Apache Arrow LIST encoding: + /// optional group {name} (LIST) { + /// repeated group list { + /// optional group element { ...elementFields } + /// } + /// } + /// + private static GroupNode ListOf(string name, params Node[] elementFields) + { + var element = new GroupNode("element", Repetition.Optional, elementFields); + var list = new GroupNode("list", Repetition.Repeated, new Node[] { element }); + return new GroupNode(name, Repetition.Optional, new Node[] { list }, LogicalType.List()); + } + + /// + /// LIST encoding where the element is a leaf primitive directly (not a wrapper + /// struct). Used for plain "list of doubles" / "list of strings" columns like + /// spectrum.mz_delta_model. + /// + private static GroupNode ListOfLeaf(string name, PrimitiveNode elementLeaf) + { + var list = new GroupNode("list", Repetition.Repeated, new Node[] { elementLeaf }); + return new GroupNode(name, Repetition.Optional, new Node[] { list }, LogicalType.List()); + } + + private static GroupNode ListOfDoubleLeaf(string name) => + ListOfLeaf(name, DoubleLeaf("element")); + + /// The 5 CV-param leaves inside a parameters list element. + private static Node[] CvParamElementFields() => new Node[] + { + StringLeaf("name"), + StringLeaf("accession"), + Struct("value", StringLeaf("string"), LongLeaf("integer"), DoubleLeaf("float"), BoolLeaf("boolean")), + StringLeaf("unit"), + StringLeaf("type"), // xsd datatype hint for userParams (e.g. "xsd:float") + }; + + private static GroupNode CvParamList(string name = "parameters") => ListOf(name, CvParamElementFields()); + + // ---------------- Spectra metadata ---------------- + + private static PrimitiveNode Int32Leaf(string name) => new(name, Repetition.Optional, LogicalType.None(), PhysicalType.Int32); + + private static GroupNode BuildSpectraMetadataSchema() => new( + "schema", Repetition.Required, + new Node[] + { + Struct("spectrum", + UInt64Leaf("index"), + StringLeaf("id"), + DoubleLeaf("time"), + Int8Leaf("MS_1000511_ms_level"), + StringLeaf("MS_1000525_spectrum_representation"), + Int8Leaf("MS_1000465_scan_polarity"), + StringLeaf("MS_1000559_spectrum_type"), + LongLeaf("MS_1003060_number_of_data_points"), + LongLeaf("MS_1003059_number_of_peaks"), + DoubleLeaf("MS_1000504_base_peak_mz_unit_MS_1000040"), + DoubleLeaf("MS_1000505_base_peak_intensity_unit_MS_1000131"), + DoubleLeaf("MS_1000285_total_ion_current_unit_MS_1000131"), + DoubleLeaf("MS_1000528_lowest_observed_mz_unit_MS_1000040"), + DoubleLeaf("MS_1000527_highest_observed_mz_unit_MS_1000040"), + CvParamList(), + StringLeaf("data_processing_ref"), + ListOfDoubleLeaf("mz_delta_model"), + Int32Leaf("number_of_auxiliary_arrays"), + StringLeaf("MS_1000570_spectrum_combination"), + ListOfLeaf("param_group_refs", StringLeaf("element")), + StringLeaf("auxiliary_arrays"), + StringLeaf("value_array_type"), + StringLeaf("value_array_unit")), + Struct("scan", + UInt64Leaf("source_index"), + UInt32Leaf("instrument_configuration_ref"), + DoubleLeaf("ion_mobility_value"), + StringLeaf("ion_mobility_type"), + DoubleLeaf("MS_1000016_scan_start_time_unit_UO_0000031"), + StringLeaf("MS_1000512_filter_string"), + LongLeaf("MS_1000616_preset_scan_configuration"), + DoubleLeaf("MS_1000927_ion_injection_time_unit_UO_0000028"), + StringLeaf("spectrum_ref"), + CvParamList(), + ListOf("scan_windows", + DoubleLeaf("MS_1000501_scan_window_lower_limit_unit_MS_1000040"), + DoubleLeaf("MS_1000500_scan_window_upper_limit_unit_MS_1000040")), + StringLeaf("scan_window_params"), + StringLeaf("extra_scans")), + Struct("precursor", + UInt64Leaf("source_index"), + UInt64Leaf("precursor_index"), + StringLeaf("precursor_id"), + Struct("isolation_window", + DoubleLeaf("MS_1000827_isolation_window_target_mz_unit_MS_1000040"), + DoubleLeaf("MS_1000828_isolation_window_lower_offset_unit_MS_1000040"), + DoubleLeaf("MS_1000829_isolation_window_upper_offset_unit_MS_1000040"), + CvParamList()), + Struct("activation", + DoubleLeaf("MS_1000045_collision_energy_unit_UO_0000266"), + StringLeaf("MS_1000044_dissociation_method"), + CvParamList())), + Struct("selected_ion", + UInt64Leaf("source_index"), + UInt64Leaf("precursor_index"), + DoubleLeaf("ion_mobility_value"), + StringLeaf("ion_mobility_type"), + DoubleLeaf("MS_1000744_selected_ion_mz_unit_MS_1000040"), + DoubleLeaf("MS_1000042_peak_intensity_unit_MS_1000131"), + LongLeaf("MS_1000041_charge_state"), + CvParamList()), + }); + + /// + /// Fan each spectrum into one row per precursor (≥1 row even when there are + /// no precursors). The first row of each spectrum is the "primary" — it + /// carries spectrum-level and scan-level columns; secondary rows have null + /// in those groups and carry only precursor + selected_ion data. Legacy + /// single-precursor fields on SpectrumToWrite synthesise one PrecursorToWrite + /// when Precursors is null. + /// + private static List BuildRows(IReadOnlyList spectra) + { + var rows = new List(spectra.Count); + foreach (var s in spectra) + { + var precs = s.Precursors; + if (precs == null || precs.Count == 0) + { + rows.Add(new Row(s, null, 0, IsPrimary: true)); + continue; + } + for (int i = 0; i < precs.Count; i++) + rows.Add(new Row(s, precs[i], i, IsPrimary: i == 0)); + } + return rows; + } + + // Group-presence predicates (passed to column writers; null outer Nested when false). + private static readonly Func Primary = r => r.IsPrimary; + private static readonly Func HasPrec = r => r.Precursor != null; + + private static void WriteSpectraMetadata(string path, IReadOnlyList spectra, FileMetadata fileMetadata) + { + var schema = BuildSpectraMetadataSchema(); + var kv = BuildKeyValueMetadata(fileMetadata, spectra.Sum(s => (long)s.Mz.Length)); + + using var props = new WriterPropertiesBuilder().Compression(Compression.Zstd).Build(); + using var fileWriter = new ParquetFileWriter(path, schema, props, keyValueMetadata: kv); + + // Split into row groups on spectrum (primary-row) boundaries so a spectrum's fan-out rows + // never span groups — lets the reader lazily load just the group(s) covering a spectrum. + var rows = BuildRows(spectra); + foreach (var (rgStart, rgLen) in ChunkRowsOnPrimary(rows)) + { + using var rg = fileWriter.AppendRowGroup(); + WriteSpectraRowGroupColumns(rg, rows.GetRange(rgStart, rgLen)); + } + fileWriter.Close(); + } + + /// Target fan-out rows per spectra_metadata row group (split only on spectrum boundaries). + private const int TargetRowsPerMetaGroup = 5000; + + private static IEnumerable<(int start, int len)> ChunkRowsOnPrimary(IReadOnlyList rows) + { + int n = rows.Count; + if (n == 0) { yield return (0, 0); yield break; } + int start = 0; + while (start < n) + { + int end = Math.Min(n, start + TargetRowsPerMetaGroup); + while (end < n && !rows[end].IsPrimary) end++; // keep a spectrum's fan-out rows together + yield return (start, end - start); + start = end; + } + } + + private static void WriteSpectraRowGroupColumns(RowGroupWriter rg, IReadOnlyList rows) + { + // -------- spectrum group (primary rows only) -------- + WriteScalar(rg, rows, Primary, r => (ulong?)r.Spectrum.Index); + WriteScalar(rg, rows, Primary, r => (string?)r.Spectrum.Id); + WriteScalar(rg, rows, Primary, r => (double?)r.Spectrum.Time); + WriteScalar(rg, rows, Primary, r => r.Spectrum.MsLevel is int m ? (sbyte?)checked((sbyte)m) : null); + WriteScalar(rg, rows, Primary, r => !r.Spectrum.HasRepresentation ? null : (r.Spectrum.IsProfile ? CurieProfileSpectrum : CurieCentroidSpectrum)); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ScanPolarity is int sp ? (sbyte?)checked((sbyte)sp) : null); + WriteScalar(rg, rows, Primary, r => r.Spectrum.SpectrumTypeCurie); + WriteScalar(rg, rows, Primary, r => (long?)(r.Spectrum.NumberOfDataPoints ?? r.Spectrum.Mz.Length)); + WriteScalar(rg, rows, Primary, r => r.Spectrum.NumberOfPeaks ?? (r.Spectrum.SupplementaryPeaksMz?.Length is int n ? (long?)n : null)); + WriteScalar(rg, rows, Primary, r => r.Spectrum.BasePeakMz); + WriteScalar(rg, rows, Primary, r => r.Spectrum.BasePeakIntensity); + WriteScalar(rg, rows, Primary, r => r.Spectrum.TotalIonCurrent); + WriteScalar(rg, rows, Primary, r => r.Spectrum.LowestObservedMz); + WriteScalar(rg, rows, Primary, r => r.Spectrum.HighestObservedMz); + WriteCvParamLeaves(rg, rows, Primary, r => r.Spectrum.SpectrumParameters); + WriteScalar(rg, rows, Primary, r => r.Spectrum.SpectrumDataProcessingRef); + WriteMzDeltaModel(rg, rows); + WriteScalar(rg, rows, Primary, r => r.Spectrum.NumberOfAuxiliaryArrays); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ScanCombinationCurie); + WriteListString(rg, rows, Primary, r => r.Spectrum.ParamGroupRefs); + WriteScalar(rg, rows, Primary, r => r.Spectrum.AuxiliaryArraysJson); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ValueArrayCurie); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ValueArrayUnitCurie); + + // -------- scan group (primary rows only) -------- + WriteScalar(rg, rows, Primary, r => (ulong?)(r.Spectrum.ScanSourceIndex ?? r.Spectrum.Index)); + WriteScalar(rg, rows, Primary, r => r.Spectrum.InstrumentConfigurationRef); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ScanIonMobilityValue); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ScanIonMobilityTypeCurie); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ScanStartTime); + WriteScalar(rg, rows, Primary, r => r.Spectrum.FilterString); + WriteScalar(rg, rows, Primary, r => r.Spectrum.PresetScanConfiguration); + WriteScalar(rg, rows, Primary, r => r.Spectrum.IonInjectionTime); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ScanSpectrumRef); + WriteCvParamLeaves(rg, rows, Primary, r => r.Spectrum.ScanParameters); + WriteListScalar(rg, rows, Primary, r => r.Spectrum.ScanWindowLowerLimits); + WriteListScalar(rg, rows, Primary, r => r.Spectrum.ScanWindowUpperLimits); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ScanWindowParamsJson); + WriteScalar(rg, rows, Primary, r => r.Spectrum.ExtraScansJson); + + // -------- precursor group (rows with a precursor) -------- + WriteScalar(rg, rows, HasPrec, r => (ulong?)r.Spectrum.Index); + WriteScalar(rg, rows, HasPrec, r => (ulong?)((ulong?)r.PrecursorIdx)); + WriteScalar(rg, rows, HasPrec, r => r.Precursor!.PrecursorId); + // isolation_window sub-struct + WriteScalar2(rg, rows, HasPrec, r => r.Precursor!.IsolationTargetMz); + WriteScalar2(rg, rows, HasPrec, r => r.Precursor!.IsolationLowerOffset); + WriteScalar2(rg, rows, HasPrec, r => r.Precursor!.IsolationUpperOffset); + WriteCvParamLeaves2(rg, rows, HasPrec, r => r.Precursor!.IsolationWindowParameters); + // activation sub-struct + WriteScalar2(rg, rows, HasPrec, r => r.Precursor!.CollisionEnergy); + WriteScalar2(rg, rows, HasPrec, r => r.Precursor!.DissociationMethodCurie); + WriteCvParamLeaves2(rg, rows, HasPrec, r => r.Precursor!.ActivationParameters); + + // -------- selected_ion group (one per precursor row) -------- + WriteScalar(rg, rows, HasPrec, r => (ulong?)r.Spectrum.Index); + WriteScalar(rg, rows, HasPrec, r => (ulong?)((ulong?)r.PrecursorIdx)); + WriteScalar(rg, rows, HasPrec, r => r.Precursor!.SelectedIonIonMobilityValue); + WriteScalar(rg, rows, HasPrec, r => r.Precursor!.SelectedIonIonMobilityTypeCurie); + WriteScalar(rg, rows, HasPrec, r => r.Precursor!.SelectedIonMz); + WriteScalar(rg, rows, HasPrec, r => r.Precursor!.SelectedIonPeakIntensity); + WriteScalar(rg, rows, HasPrec, r => r.Precursor!.SelectedIonChargeState); + WriteCvParamLeaves(rg, rows, HasPrec, r => r.Precursor!.SelectedIonParameters); + } + + // ---------------- Spectra binary data (canonical + supplementary peaks) ---------------- + + private static GroupNode BuildPointSchema(string parentIndexName, string valueColumnName) => new( + "schema", Repetition.Required, + new Node[] + { + Struct("point", + UInt64Leaf(parentIndexName), + DoubleLeaf(valueColumnName), + FloatLeaf("intensity")), + }); + + private static void WriteSpectraData(string path, IReadOnlyList spectra) + => WritePointLayout(path, "spectrum_index", "mz", "spectrum_array_index", BuildSpectraDataArrayIndex(), + spectra.SelectMany(s => Enumerable.Range(0, s.Mz.Length).Select(i => ((ulong?)s.Index, (double?)s.Mz[i], (float?)s.Intensity[i])))); + + private static void WriteSpectraPeaks(string path, IReadOnlyList spectra) + => WritePointLayout(path, "spectrum_index", "mz", "spectrum_array_index", BuildSpectraDataArrayIndex(), + spectra + .Where(s => s.SupplementaryPeaksMz is not null) + .SelectMany(s => Enumerable.Range(0, s.SupplementaryPeaksMz!.Length) + .Select(i => ((ulong?)s.Index, (double?)s.SupplementaryPeaksMz![i], (float?)s.SupplementaryPeaksIntensity![i])))); + + /// + /// "spectrum_array_index" / "chromatogram_array_index" KV blocks tell consumers how to + /// interpret the point-layout columns: which CV term identifies the mz vs intensity + /// data, the prefix used in the parquet schema, etc. Required by mzPeak.NET's + /// DataArraysReaderMeta to open the data file. + /// + private static string BuildSpectraDataArrayIndex() => System.Text.Json.JsonSerializer.Serialize(new + { + prefix = "point", + entries = new object[] + { + new { context = "spectrum", path = "point.mz", data_type = "MS:1000523", array_type = "MS:1000514", array_name = "m/z array", unit = "MS:1000040" }, + new { context = "spectrum", path = "point.intensity", data_type = "MS:1000521", array_type = "MS:1000515", array_name = "intensity array", unit = "MS:1000131" }, + }, + }); + + private static string BuildChromatogramsDataArrayIndex() => System.Text.Json.JsonSerializer.Serialize(new + { + prefix = "point", + entries = new object[] + { + new { context = "chromatogram", path = "point.time", data_type = "MS:1000523", array_type = "MS:1000595", array_name = "time array", unit = "UO:0000031" }, + new { context = "chromatogram", path = "point.intensity", data_type = "MS:1000521", array_type = "MS:1000515", array_name = "intensity array", unit = "MS:1000131" }, + }, + }); + + /// + /// Target points per row group in the point-layout data files. Row groups are split only on + /// parent-index (spectrum/chromatogram) boundaries so a parent's points never span groups — + /// that lets the reader lazily load just the row group(s) covering a requested parent. The + /// per-row-group parent-index range is recorded in the point_row_group_ranges KV. + /// + private const int TargetPointsPerRowGroup = 1_000_000; + + private const string PointRowGroupRangesKey = "point_row_group_ranges"; + + private static void WritePointLayout(string path, string parentIndexName, string valueColumnName, string arrayIndexKey, string arrayIndexJson, IEnumerable<(ulong? idx, double? value, float? intensity)> rows) + { + var schema = BuildPointSchema(parentIndexName, valueColumnName); + var (idxArr, valArr, intArr) = MaterializeTriple(rows); + int total = idxArr.Length; + + // Compute row-group boundaries: ~TargetPointsPerRowGroup points each, extended so a parent's + // contiguous points are never split across groups. + var bounds = new List<(int start, int len)>(); + var ranges = new List(); + int rgStart = 0; + while (rgStart < total) + { + int rgEnd = Math.Min(total, rgStart + TargetPointsPerRowGroup); + while (rgEnd < total && idxArr[rgEnd] == idxArr[rgEnd - 1]) rgEnd++; + bounds.Add((rgStart, rgEnd - rgStart)); + ranges.Add(new[] { (long)(idxArr[rgStart] ?? 0), (long)(idxArr[rgEnd - 1] ?? 0) }); + rgStart = rgEnd; + } + if (bounds.Count == 0) { bounds.Add((0, 0)); ranges.Add(new long[] { -1, -1 }); } + + var kv = new Dictionary + { + [arrayIndexKey] = arrayIndexJson, + [PointRowGroupRangesKey] = System.Text.Json.JsonSerializer.Serialize(ranges), + }; + using var props = new WriterPropertiesBuilder().Compression(Compression.Zstd).Build(); + using var fileWriter = new ParquetFileWriter(path, schema, props, keyValueMetadata: kv); + foreach (var (start, len) in bounds) + { + using var rg = fileWriter.AppendRowGroup(); + // Leaves live inside the "point" group → 1 Nested wrap. + WriteNestedScalar1Slice(rg, idxArr, start, len); + WriteNestedScalar1Slice(rg, valArr, start, len); + WriteNestedScalar1Slice(rg, intArr, start, len); + } + fileWriter.Close(); + } + + private static void WriteNestedScalar1Slice(RowGroupWriter rg, T?[] values, int start, int len) where T : struct + { + using var w = rg.NextColumn().LogicalWriter?>(); + var arr = new Nested?[len]; + for (int i = 0; i < len; i++) arr[i] = new Nested(values[start + i]); + w.WriteBatch(arr); + } + + private static void WriteNestedScalar1(RowGroupWriter rg, T?[] values) where T : struct + { + using var w = rg.NextColumn().LogicalWriter?>(); + var arr = new Nested?[values.Length]; + for (int i = 0; i < values.Length; i++) arr[i] = new Nested(values[i]); + w.WriteBatch(arr); + } + + private static (ulong?[], double?[], float?[]) MaterializeTriple(IEnumerable<(ulong? idx, double? value, float? intensity)> rows) + { + var list = rows as IList<(ulong? idx, double? value, float? intensity)> ?? rows.ToList(); + int n = list.Count; + var idx = new ulong?[n]; var val = new double?[n]; var inten = new float?[n]; + for (int i = 0; i < n; i++) { idx[i] = list[i].idx; val[i] = list[i].value; inten[i] = list[i].intensity; } + return (idx, val, inten); + } + + // ---------------- Chromatograms ---------------- + + private static GroupNode BuildChromatogramsMetadataSchema() => new( + "schema", Repetition.Required, + new Node[] + { + Struct("chromatogram", + UInt64Leaf("index"), + StringLeaf("id"), + StringLeaf("MS_1000626_chromatogram_type"), + StringLeaf("data_processing_ref"), + StringLeaf("MS_1000595_time_array_unit"), + StringLeaf("MS_1000515_intensity_array_unit"), + CvParamList(), + StringLeaf("auxiliary_arrays")), + // mzPeak.NET's ChromatogramMetadataReader unconditionally calls + // batch.Column("precursor") / batch.Column("selected_ion") on the + // chromatograms_metadata file. The demo populates both for SIM-style + // chromatograms. We currently don't write any precursor/selected_ion + // data per chromatogram, so emit empty struct groups with a single + // null-valued field each just to satisfy mzPeak.NET's column lookup. + Struct("precursor", UInt64Leaf("source_index")), + Struct("selected_ion", UInt64Leaf("source_index")), + }); + + private static void WriteChromatogramsMetadata(string path, IReadOnlyList chromatograms) + { + var schema = BuildChromatogramsMetadataSchema(); + using var props = new WriterPropertiesBuilder().Compression(Compression.Zstd).Build(); + using var fileWriter = new ParquetFileWriter(path, schema, props); + using var rg = fileWriter.AppendRowGroup(); + // Leaves under the "chromatogram" group — 1-deep Nested wrap. + WriteNestedScalar1(rg, chromatograms.Select(c => (ulong?)c.Index).ToArray()); + WriteChromString(rg, chromatograms, c => c.Id); + WriteChromString(rg, chromatograms, c => c.ChromatogramTypeCurie); + WriteChromString(rg, chromatograms, c => c.DataProcessingRef); + WriteChromString(rg, chromatograms, c => c.TimeUnitCurie); + WriteChromString(rg, chromatograms, c => c.IntensityUnitCurie); + WriteChromCvParamLeaves(rg, chromatograms, c => c.Parameters); + WriteChromString(rg, chromatograms, c => c.AuxiliaryArraysJson); + // precursor.source_index — emit nulls (we don't track per-chromatogram precursors yet) + WriteNestedScalar1(rg, new ulong?[chromatograms.Count]); + // selected_ion.source_index — same + WriteNestedScalar1(rg, new ulong?[chromatograms.Count]); + fileWriter.Close(); + } + + private static void WriteChromString(RowGroupWriter rg, IReadOnlyList chroms, Func selector) + { + using var w = rg.NextColumn().LogicalWriter?>(); + var arr = new Nested?[chroms.Count]; + for (int i = 0; i < chroms.Count; i++) arr[i] = new Nested(selector(chroms[i])); + w.WriteBatch(arr); + } + + // Chromatogram-level CvParam list — same 8 leaves as the spectrum/scan param lists, but one + // optional group deep (the chromatogram struct), always present. + private static void WriteChromCvParamLeaves(RowGroupWriter rg, IReadOnlyList chroms, + Func?> getter) + { + // name/accession/unit/type are direct element fields (one group level above the list); + // value.* live inside the nested `value` struct, so they need one extra Nested level. + WriteChromListedString(rg, chroms, getter, p => p.Name); + WriteChromListedString(rg, chroms, getter, p => p.Accession); + WriteChromListedStringValue(rg, chroms, getter, p => p.ValueString); + WriteChromListedValue(rg, chroms, getter, p => p.ValueInteger); + WriteChromListedValue(rg, chroms, getter, p => p.ValueFloat); + WriteChromListedValue(rg, chroms, getter, p => p.ValueBoolean); + WriteChromListedString(rg, chroms, getter, p => p.Unit); + WriteChromListedString(rg, chroms, getter, p => p.Type); + } + + // Direct element field (name / accession / unit / type): chromatogram-group → list → element-field. + private static void WriteChromListedString(RowGroupWriter rg, IReadOnlyList chroms, + Func?> getter, Func selector) + { + using var w = rg.NextColumn().LogicalWriter?[]>?>(); + var arr = new Nested?[]>?[chroms.Count]; + for (int i = 0; i < chroms.Count; i++) + { + var p = getter(chroms[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>() + : p.Select(x => (Nested?)new Nested(selector(x))).ToArray(); + arr[i] = new Nested?[]>(elems); + } + w.WriteBatch(arr); + } + + // value.string leaf: one extra Nested for the `value` sub-struct. + private static void WriteChromListedStringValue(RowGroupWriter rg, IReadOnlyList chroms, + Func?> getter, Func selector) + { + using var w = rg.NextColumn().LogicalWriter?>?[]>?>(); + var arr = new Nested?>?[]>?[chroms.Count]; + for (int i = 0; i < chroms.Count; i++) + { + var p = getter(chroms[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>?>() + : p.Select(x => (Nested?>?)new Nested?>(new Nested(selector(x)))).ToArray(); + arr[i] = new Nested?>?[]>(elems); + } + w.WriteBatch(arr); + } + + // value.{integer,float,boolean} leaf: one extra Nested for the `value` sub-struct. + private static void WriteChromListedValue(RowGroupWriter rg, IReadOnlyList chroms, + Func?> getter, Func selector) + where T : unmanaged + { + using var w = rg.NextColumn().LogicalWriter?>?[]>?>(); + var arr = new Nested?>?[]>?[chroms.Count]; + for (int i = 0; i < chroms.Count; i++) + { + var p = getter(chroms[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>?>() + : p.Select(x => (Nested?>?)new Nested?>(new Nested(selector(x)))).ToArray(); + arr[i] = new Nested?>?[]>(elems); + } + w.WriteBatch(arr); + } + + private static void WriteChromatogramsData(string path, IReadOnlyList chromatograms) + => WritePointLayout(path, "chromatogram_index", "time", "chromatogram_array_index", BuildChromatogramsDataArrayIndex(), + chromatograms.SelectMany(c => Enumerable.Range(0, c.Time.Length).Select(i => ((ulong?)c.Index, (double?)c.Time[i], (float?)c.Intensity[i])))); + + // ---------------- Write helpers ---------------- + + // ===== Column writers — operate over the fan-out Row list ===== + // + // Every entity group (spectrum, scan, precursor, selected_ion) is optional + // in the parquet schema. The "present" predicate gates the outer Nested: + // when it returns false (e.g. a non-primary row for the spectrum group), + // the outer wrapper is null, which serialises as "this group is absent". + + private static void WriteScalar(RowGroupWriter rg, IReadOnlyList rows, Func present, Func selector) + where T : struct + { + using var w = rg.NextColumn().LogicalWriter?>(); + var arr = new Nested?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + arr[i] = present(rows[i]) ? new Nested(selector(rows[i])) : null; + w.WriteBatch(arr); + } + + private static void WriteScalar(RowGroupWriter rg, IReadOnlyList rows, Func present, Func selector) + { + using var w = rg.NextColumn().LogicalWriter?>(); + var arr = new Nested?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + arr[i] = present(rows[i]) ? new Nested(selector(rows[i])) : null; + w.WriteBatch(arr); + } + + // Two optional groups deep — outer wraps the entity group, inner wraps the + // sub-struct (e.g. precursor.isolation_window). Both gate on `present`. + private static void WriteScalar2(RowGroupWriter rg, IReadOnlyList rows, Func present, Func selector) + where T : struct + { + using var w = rg.NextColumn().LogicalWriter?>?>(); + var arr = new Nested?>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + arr[i] = present(rows[i]) ? new Nested?>(new Nested(selector(rows[i]))) : null; + w.WriteBatch(arr); + } + + private static void WriteScalar2(RowGroupWriter rg, IReadOnlyList rows, Func present, Func selector) + { + using var w = rg.NextColumn().LogicalWriter?>?>(); + var arr = new Nested?>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + arr[i] = present(rows[i]) ? new Nested?>(new Nested(selector(rows[i]))) : null; + w.WriteBatch(arr); + } + + // List of leaf doubles inside one optional group (spectrum.mz_delta_model). + private static void WriteMzDeltaModel(RowGroupWriter rg, IReadOnlyList rows) + { + using var w = rg.NextColumn().LogicalWriter?>(); + var arr = new Nested?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + arr[i] = rows[i].IsPrimary + ? new Nested(rows[i].Spectrum.MzDeltaModel ?? Array.Empty()) + : null; + w.WriteBatch(arr); + } + + // List-of-leaf (plain strings) inside one optional group, mirroring WriteMzDeltaModel's + // Nested shape (no inner element-struct wrap). + private static void WriteListString(RowGroupWriter rg, IReadOnlyList rows, Func present, Func?> selector) + { + using var w = rg.NextColumn().LogicalWriter?>(); + var arr = new Nested?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var src = selector(rows[i]); + arr[i] = new Nested(src is null || src.Count == 0 + ? Array.Empty() + : src.Select(x => (string?)x).ToArray()); + } + w.WriteBatch(arr); + } + + // scan_window-style list inside one optional group, with element-struct wrap. + private static void WriteListScalar(RowGroupWriter rg, IReadOnlyList rows, Func present, Func selector) + where T : unmanaged + { + using var w = rg.NextColumn().LogicalWriter?[]>?>(); + var arr = new Nested?[]>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var src = selector(rows[i]); + var elems = (src is null || src.Length == 0) + ? Array.Empty?>() + : src.Select(x => (Nested?)new Nested(x)).ToArray(); + arr[i] = new Nested?[]>(elems); + } + w.WriteBatch(arr); + } + + // All 7 leaves of a CvParam list inside one optional entity group. + private static void WriteCvParamLeaves( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter) + { + WriteListedStringLeaf1(rg, rows, present, getter, p => p.Name); + WriteListedStringLeaf1(rg, rows, present, getter, p => p.Accession); + WriteListedStringLeaf1Value(rg, rows, present, getter, p => p.ValueString); + WriteListedValueLeaf1Value(rg, rows, present, getter, p => p.ValueInteger); + WriteListedValueLeaf1Value(rg, rows, present, getter, p => p.ValueFloat); + WriteListedValueLeaf1Value(rg, rows, present, getter, p => p.ValueBoolean); + WriteListedStringLeaf1(rg, rows, present, getter, p => p.Unit); + WriteListedStringLeaf1(rg, rows, present, getter, p => p.Type); + } + + // Same shape, two outer groups deep (precursor.isolation_window.parameters etc.). + private static void WriteCvParamLeaves2( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter) + { + WriteListedStringLeaf2(rg, rows, present, getter, p => p.Name); + WriteListedStringLeaf2(rg, rows, present, getter, p => p.Accession); + WriteListedStringLeaf2Value(rg, rows, present, getter, p => p.ValueString); + WriteListedValueLeaf2Value(rg, rows, present, getter, p => p.ValueInteger); + WriteListedValueLeaf2Value(rg, rows, present, getter, p => p.ValueFloat); + WriteListedValueLeaf2Value(rg, rows, present, getter, p => p.ValueBoolean); + WriteListedStringLeaf2(rg, rows, present, getter, p => p.Unit); + WriteListedStringLeaf2(rg, rows, present, getter, p => p.Type); + } + + private static void WriteListedStringLeaf1( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter, + Func selector) + { + using var w = rg.NextColumn().LogicalWriter?[]>?>(); + var arr = new Nested?[]>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var p = getter(rows[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>() + : p.Select(x => (Nested?)new Nested(selector(x))).ToArray(); + arr[i] = new Nested?[]>(elems); + } + w.WriteBatch(arr); + } + + private static void WriteListedStringLeaf1Value( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter, + Func selector) + { + using var w = rg.NextColumn().LogicalWriter?>?[]>?>(); + var arr = new Nested?>?[]>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var p = getter(rows[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>?>() + : p.Select(x => (Nested?>?)new Nested?>(new Nested(selector(x)))).ToArray(); + arr[i] = new Nested?>?[]>(elems); + } + w.WriteBatch(arr); + } + + private static void WriteListedValueLeaf1Value( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter, + Func selector) where T : unmanaged + { + using var w = rg.NextColumn().LogicalWriter?>?[]>?>(); + var arr = new Nested?>?[]>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var p = getter(rows[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>?>() + : p.Select(x => (Nested?>?)new Nested?>(new Nested(selector(x)))).ToArray(); + arr[i] = new Nested?>?[]>(elems); + } + w.WriteBatch(arr); + } + + private static void WriteListedStringLeaf2( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter, + Func selector) + { + using var w = rg.NextColumn().LogicalWriter?[]>?>?>(); + var arr = new Nested?[]>?>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var p = getter(rows[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>() + : p.Select(x => (Nested?)new Nested(selector(x))).ToArray(); + arr[i] = new Nested?[]>?>(new Nested?[]>(elems)); + } + w.WriteBatch(arr); + } + + private static void WriteListedStringLeaf2Value( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter, + Func selector) + { + using var w = rg.NextColumn().LogicalWriter?>?[]>?>?>(); + var arr = new Nested?>?[]>?>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var p = getter(rows[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>?>() + : p.Select(x => (Nested?>?)new Nested?>(new Nested(selector(x)))).ToArray(); + arr[i] = new Nested?>?[]>?>(new Nested?>?[]>(elems)); + } + w.WriteBatch(arr); + } + + private static void WriteListedValueLeaf2Value( + RowGroupWriter rg, IReadOnlyList rows, Func present, + Func?> getter, + Func selector) where T : unmanaged + { + using var w = rg.NextColumn().LogicalWriter?>?[]>?>?>(); + var arr = new Nested?>?[]>?>?[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + if (!present(rows[i])) { arr[i] = null; continue; } + var p = getter(rows[i]); + var elems = (p is null || p.Count == 0) + ? Array.Empty?>?>() + : p.Select(x => (Nested?>?)new Nested?>(new Nested(selector(x)))).ToArray(); + arr[i] = new Nested?>?[]>?>(new Nested?>?[]>(elems)); + } + w.WriteBatch(arr); + } + + // ---------------- File-level metadata (KV JSON) + manifest ---------------- + + private static void WriteManifest(string path, bool includeChromatograms, bool includeSupplementaryPeaks) + { + var files = new List + { + new { name = "spectra_metadata.parquet", entity_type = "spectrum", data_kind = "metadata" }, + new { name = "spectra_data.parquet", entity_type = "spectrum", data_kind = "data arrays" }, + }; + if (includeSupplementaryPeaks) + files.Add(new { name = "spectra_peaks.parquet", entity_type = "spectrum", data_kind = "peaks" }); + if (includeChromatograms) + { + files.Add(new { name = "chromatograms_metadata.parquet", entity_type = "chromatogram", data_kind = "metadata" }); + files.Add(new { name = "chromatograms_data.parquet", entity_type = "chromatogram", data_kind = "data arrays" }); + } + var manifest = new { files = files.ToArray(), metadata = new { } }; + File.WriteAllText(path, JsonSerializer.Serialize(manifest, ManifestJsonOpts)); + } + + private static readonly JsonSerializerOptions ManifestJsonOpts = new() { WriteIndented = true }; + + private static IReadOnlyDictionary BuildKeyValueMetadata(FileMetadata fm, long spectraDataPointCount) + { + var opts = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + var kv = new Dictionary + { + ["file_description"] = JsonSerializer.Serialize(fm.FileDescription, opts), + ["instrument_configuration_list"] = JsonSerializer.Serialize(fm.InstrumentConfigurations, opts), + ["data_processing_method_list"] = JsonSerializer.Serialize(fm.DataProcessingMethods, opts), + ["software_list"] = JsonSerializer.Serialize(fm.Software, opts), + ["sample_list"] = JsonSerializer.Serialize(fm.Samples, opts), + ["run"] = JsonSerializer.Serialize(fm.Run, opts), + ["spectrum_count"] = "0", + ["spectrum_data_point_count"] = spectraDataPointCount.ToString(System.Globalization.CultureInfo.InvariantCulture), + }; + if (!string.IsNullOrEmpty(fm.DocumentId)) + kv["document_id"] = fm.DocumentId!; + if (fm.ParamGroups is { Count: > 0 }) + kv["referenceable_param_group_list"] = JsonSerializer.Serialize(fm.ParamGroups, opts); + return kv; + } +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/SpectrumList_MzPeak.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/SpectrumList_MzPeak.cs new file mode 100644 index 00000000000..9d3dbb0daa7 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/SpectrumList_MzPeak.cs @@ -0,0 +1,335 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.Common.Params; +using Pwiz.Data.MsData.Processing; +using Pwiz.Data.MsData.Spectra; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// Lazy over an . The +/// reader already buffers every metadata column in memory at open time, so +/// "lazy" here means "translate columns to objects on +/// demand" — there's no extra parquet I/O per call. Binary data +/// (m/z + intensity arrays) is still pulled from the per-spectrum buckets the +/// reader built up at construction. +/// +internal sealed class SpectrumList_MzPeak : SpectrumListBase +{ + private readonly MzPeakReader _reader; + private readonly bool _ownsReader; + private readonly DataProcessing? _dp; + private readonly SpectrumIdentity[] _identities; + // referenceableParamGroups referenced by spectra, keyed by id, plus the set of CVIDs each + // provides (so params a referenced group already carries aren't also inlined as direct params). + private readonly Dictionary _paramGroupsById = new(StringComparer.Ordinal); + private readonly Dictionary> _paramGroupCvids = new(StringComparer.Ordinal); + + public SpectrumList_MzPeak(MzPeakReader reader, DataProcessing? dp, bool ownsReader, + IReadOnlyList? paramGroups = null) + { + ArgumentNullException.ThrowIfNull(reader); + _reader = reader; + _ownsReader = ownsReader; + _dp = dp; + if (paramGroups is not null) + foreach (var pg in paramGroups) + { + _paramGroupsById[pg.Id] = pg; + _paramGroupCvids[pg.Id] = pg.CVParams.Select(cv => cv.Cvid).ToHashSet(); + } + _identities = new SpectrumIdentity[reader.SpectrumCount]; + for (int i = 0; i < reader.SpectrumCount; i++) + { + // Use the lightweight id accessor — building identities must not force a full lazy + // metadata-group load for every spectrum at open. + _identities[i] = new SpectrumIdentity { Index = i, Id = reader.GetSpectrumId(i) }; + } + } + + public override int Count => _identities.Length; + + public override SpectrumIdentity SpectrumIdentity(int index) => _identities[index]; + + public override DataProcessing? DataProcessing => _dp; + + public override Spectrum GetSpectrum(int index, bool getBinaryData = false) + { + if ((uint)index >= (uint)_identities.Length) + throw new ArgumentOutOfRangeException(nameof(index)); + + var desc = _reader.GetSpectrumDescription(index); + var spectrum = new Spectrum + { + Index = index, + Id = desc.Id, + }; + + TranslateSpectrumLevel(desc, spectrum); + TranslateScan(desc, spectrum); + TranslatePrecursors(desc, spectrum); + ApplyScanCombination(desc, spectrum); + ApplyParamGroupRefs(desc, spectrum); + + if (getBinaryData) + { + // pwiz/mzML spectra always carry an m/z + intensity array pair, even when empty + // (a 0-point spectrum still serializes two zero-length arrays). Emit them + // unconditionally so empty spectra round-trip with the right array shape. + var data = _reader.GetSpectrumData(index); + var mz = data?.Mz ?? Array.Empty(); + var intensity = data?.Intensity ?? Array.Empty(); + // mzPeak stores intensities as float for size; pwiz's binary arrays are double — + // widen here. Unit is "detector counts" matching what every other reader emits when + // the vendor doesn't tag a more specific unit. + var intensityDouble = new double[intensity.Length]; + for (int i = 0; i < intensity.Length; i++) intensityDouble[i] = intensity[i]; + if (desc.ValueArrayCurie is null) + { + // Common case: the value array is m/z. + spectrum.SetMZIntensityArrays(mz, intensityDouble, CVID.MS_number_of_detector_counts); + } + else + { + // Non-m/z value array (e.g. UV/DAD wavelength): rebuild with its real type + unit. + var valueCvid = CvidFromCurie(desc.ValueArrayCurie); + var valueUnit = CvidFromCurie(desc.ValueArrayUnitCurie); + var valueArr = new Pwiz.Data.MsData.Spectra.BinaryDataArray(); + valueArr.Set(valueCvid, "", valueUnit); + valueArr.Data.AddRange(mz); + spectrum.BinaryDataArrays.Add(valueArr); + var intArr = new Pwiz.Data.MsData.Spectra.BinaryDataArray(); + intArr.Set(CVID.MS_intensity_array, "", CVID.MS_number_of_detector_counts); + intArr.Data.AddRange(intensityDouble); + spectrum.BinaryDataArrays.Add(intArr); + spectrum.DefaultArrayLength = mz.Length; + } + MzPeakAuxArrays.Apply(desc.AuxArrays, spectrum.BinaryDataArrays, spectrum.IntegerDataArrays); + } + else if (desc.NumberOfDataPoints is long n) + { + // Caller asked for metadata-only — keep the array-length hint so + // downstream code can decide how much to allocate later. + spectrum.DefaultArrayLength = checked((int)n); + } + + return spectrum; + } + + protected override void DisposeCore() + { + if (_ownsReader) _reader.Dispose(); + } + + // ===== Translation: mzPeak columns → pwiz MSData params ===== + + private static void TranslateSpectrumLevel(MzPeakReader.SpectrumDescription desc, Spectrum spectrum) + { + // Representation: profile vs centroid. cpp writers emit one of these + // unconditionally; we mirror that so downstream code that branches on + // HasCVParam(MS_centroid_spectrum) sees something sensible. + if (desc.IsProfile) spectrum.Params.Set(CVID.MS_profile_spectrum); + else if (desc.IsCentroid) spectrum.Params.Set(CVID.MS_centroid_spectrum); + + if (desc.MsLevel is int msLevel) spectrum.Params.Set(CVID.MS_ms_level, msLevel); + + // mzPeak's scan_polarity column is encoded as MSData's Int8 +1/-1 + // (per CV MS:1000465). pwiz emits the polarity CV terms directly. + switch (desc.ScanPolarity) + { + case 1: spectrum.Params.Set(CVID.MS_positive_scan); break; + case -1: spectrum.Params.Set(CVID.MS_negative_scan); break; + } + + if (desc.BasePeakMz is double bpmz) spectrum.Params.Set(CVID.MS_base_peak_m_z, bpmz, CVID.MS_m_z); + if (desc.BasePeakIntensity is double bpi) + spectrum.Params.Set(CVID.MS_base_peak_intensity, bpi, CVID.MS_number_of_detector_counts); + if (desc.TotalIonCurrent is double tic) + // mzML emits total ion current unitless (matching pwiz cpp output); don't synthesize a unit. + spectrum.Params.Set(CVID.MS_total_ion_current, tic); + if (desc.LowestObservedMz is double lmz) + spectrum.Params.Set(CVID.MS_lowest_observed_m_z, lmz, CVID.MS_m_z); + if (desc.HighestObservedMz is double hmz) + spectrum.Params.Set(CVID.MS_highest_observed_m_z, hmz, CVID.MS_m_z); + + ApplyParams(spectrum.Params, desc.Parameters); + } + + private static void TranslateScan(MzPeakReader.SpectrumDescription desc, Spectrum spectrum) + { + if (desc.Scan is not { } src) return; + + var scan = new Scan(); + if (!string.IsNullOrEmpty(src.SpectrumRef)) scan.SpectrumId = src.SpectrumRef!; + if (src.StartTime is double st) + scan.Set(CVID.MS_scan_start_time, st, CVID.UO_minute); + if (!string.IsNullOrEmpty(src.FilterString)) + scan.Set(CVID.MS_filter_string, src.FilterString!); + if (src.IonInjectionTime is double iit) + scan.Set(CVID.MS_ion_injection_time, iit, CVID.UO_millisecond); + if (src.PresetScanConfiguration is long psc) + scan.Set(CVID.MS_preset_scan_configuration, psc); + + // Ion mobility — the type CURIE tells us which CV term to write the + // value under (drift time, reverse drift time, FAIMS compensation V, …). + // Translate by accession; unknown types are emitted as a user param so + // round-trip survives even when the CV table doesn't know the term. + if (src.IonMobilityValue is double im) + { + var imCvid = CvidFromCurie(src.IonMobilityTypeCurie); + if (imCvid != CVID.CVID_Unknown) + scan.Set(imCvid, im); + else + scan.UserParams.Add(new Pwiz.Data.Common.Params.UserParam( + "ion mobility value", im.ToString(System.Globalization.CultureInfo.InvariantCulture))); + } + + var windowParams = ScanWindowParams.Parse(src.ScanWindowParamsJson); + for (int wi = 0; wi < src.ScanWindows.Count; wi++) + { + var win = src.ScanWindows[wi]; + if (win.LowerLimit is double lo && win.UpperLimit is double hi) + { + var sw = new ScanWindow(lo, hi, CVID.MS_m_z); + if (wi < windowParams.Count) MzPeakAuxArrays.ApplyMzPeakParams(sw, windowParams[wi]); + scan.ScanWindows.Add(sw); + } + } + + ApplyParams(scan, src.Parameters); + + spectrum.ScanList.Scans.Add(scan); + + // Append scanList scans beyond the first (combined ion-mobility spectra: one per mobility bin). + foreach (var extra in ExtraScans.Parse(src.ExtraScansJson)) + { + var es = new Scan(); + if (!string.IsNullOrEmpty(extra.SpectrumId)) es.SpectrumId = extra.SpectrumId!; + MzPeakAuxArrays.ApplyMzPeakParams(es, extra.Params); + foreach (var win in extra.ScanWindows) + { + var sw = new ScanWindow(); + MzPeakAuxArrays.ApplyMzPeakParams(sw, win); + es.ScanWindows.Add(sw); + } + spectrum.ScanList.Scans.Add(es); + } + } + + /// + /// Restore the scanList combination method (MS:1000570 children). pwiz/mzML always carries + /// one on the scanList; we emit whatever the writer captured. + /// + private static void ApplyScanCombination(MzPeakReader.SpectrumDescription desc, Spectrum spectrum) + { + var cvid = CvidFromCurie(desc.ScanCombinationCurie); + if (cvid != CVID.CVID_Unknown) + spectrum.ScanList.Params.Set(cvid); + } + + /// + /// Re-attach the referenceableParamGroups this spectrum referenced. Any CV term a referenced + /// group already provides is dropped from the spectrum's direct params so it isn't duplicated + /// (the writer inlines polarity / representation from typed columns even when the group carried + /// them). + /// + private void ApplyParamGroupRefs(MzPeakReader.SpectrumDescription desc, Spectrum spectrum) + { + if (desc.ParamGroupRefs is null) return; + foreach (var refId in desc.ParamGroupRefs) + { + if (!_paramGroupsById.TryGetValue(refId, out var pg)) continue; + spectrum.Params.ParamGroups.Add(pg); + if (_paramGroupCvids.TryGetValue(refId, out var cvids)) + spectrum.Params.CVParams.RemoveAll(cv => cvids.Contains(cv.Cvid)); + } + } + + private static void TranslatePrecursors(MzPeakReader.SpectrumDescription desc, Spectrum spectrum) + { + foreach (var p in desc.Precursors) + { + var precursor = new Precursor(); + if (!string.IsNullOrEmpty(p.PrecursorId)) + precursor.SpectrumId = p.PrecursorId!; + + if (p.IsolationWindow is { } iso) + { + if (iso.TargetMz is double t) precursor.IsolationWindow.Set(CVID.MS_isolation_window_target_m_z, t, CVID.MS_m_z); + if (iso.LowerOffset is double lo) precursor.IsolationWindow.Set(CVID.MS_isolation_window_lower_offset, lo, CVID.MS_m_z); + if (iso.UpperOffset is double hi) precursor.IsolationWindow.Set(CVID.MS_isolation_window_upper_offset, hi, CVID.MS_m_z); + ApplyParams(precursor.IsolationWindow, iso.Parameters); + } + + if (p.Activation is { } act) + { + if (act.CollisionEnergy is double ce) + precursor.Activation.Set(CVID.MS_collision_energy, ce, CVID.UO_electronvolt); + var dissCvid = CvidFromCurie(act.DissociationMethod); + if (dissCvid != CVID.CVID_Unknown) + precursor.Activation.Set(dissCvid); + ApplyParams(precursor.Activation, act.Parameters); + } + + if (p.SelectedIon is { } si) + { + var selectedIon = new SelectedIon(); + if (si.Mz is double mz) + selectedIon.Set(CVID.MS_selected_ion_m_z, mz, CVID.MS_m_z); + if (si.PeakIntensity is double pi) + // Selected-ion peak intensity is emitted unitless in mzML; don't synthesize a unit. + selectedIon.Set(CVID.MS_peak_intensity, pi); + if (si.ChargeState is long cs) + selectedIon.Set(CVID.MS_charge_state, cs); + ApplyParams(selectedIon, si.Parameters); + precursor.SelectedIons.Add(selectedIon); + } + + spectrum.Precursors.Add(precursor); + } + } + + /// + /// Apply a list of free-form s onto a + /// pwiz . CV-tagged params (accession matches a + /// known CVID) become CVParams; anything else becomes a UserParam so it + /// round-trips even if the term is unknown to this build. + /// + private static void ApplyParams(ParamContainer target, IReadOnlyList src) + { + foreach (var p in src) + { + var cvid = CvidFromCurie(p.Accession); + var unitCvid = CVID.CVID_Unknown; + if (!string.IsNullOrEmpty(p.Unit)) unitCvid = CvidFromCurie(p.Unit); + string value = ScalarToString(p); + + if (cvid != CVID.CVID_Unknown) + { + target.Set(cvid, value, unitCvid); + } + else + { + target.UserParams.Add(new Pwiz.Data.Common.Params.UserParam( + p.Name ?? string.Empty, value, type: p.Type ?? string.Empty, units: unitCvid)); + } + } + } + + private static string ScalarToString(MzPeakReader.CvParam p) + { + if (p.ValueString is not null) return p.ValueString; + if (p.ValueInteger is long li) return li.ToString(System.Globalization.CultureInfo.InvariantCulture); + if (p.ValueFloat is double d) return d.ToString("R", System.Globalization.CultureInfo.InvariantCulture); + if (p.ValueBoolean is bool b) return b ? "true" : "false"; + return string.Empty; + } + + private static CVID CvidFromCurie(string? curie) + { + if (string.IsNullOrEmpty(curie)) return CVID.CVID_Unknown; + return CvLookup.CvTermInfo(curie).Cvid; + } +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeak/WriterMzPeak.cs b/pwiz-sharp/pwiz/src/MsData/MzPeak/WriterMzPeak.cs new file mode 100644 index 00000000000..455bb969116 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeak/WriterMzPeak.cs @@ -0,0 +1,761 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.Common.Params; +using Pwiz.Data.MsData.Instruments; +using Pwiz.Data.MsData.Spectra; +using MzPeakFileMetadata = Pwiz.Data.MsData.MzPeak.FileMetadata; +using MzPeakSourceFile = Pwiz.Data.MsData.MzPeak.SourceFile; +using MzPeakFileDescription = Pwiz.Data.MsData.MzPeak.FileDescription; +using MzPeakInstrumentConfiguration = Pwiz.Data.MsData.MzPeak.InstrumentConfiguration; + +namespace Pwiz.Data.MsData.MzPeak; + +/// +/// Writes an document to the mzPeak Parquet-archive format. +/// The inverse of + MzPeakReaderAdapter: +/// MSData spectra / chromatograms / file-level metadata are translated into the +/// row-shaped records the column-oriented consumes, +/// then handed off in one call. Round-trip-safe for the columns the reader +/// understands; CV params unknown to the writer's schema land as free-form +/// CV params under spectrum/scan/precursor parameter lists. +/// +public sealed class WriterMzPeak +{ + /// Write the MSData to the given .mzpeak path. Overwrites any existing file. + public static void Write(MSData msd, string outputPath) + { + ArgumentNullException.ThrowIfNull(msd); + ArgumentException.ThrowIfNullOrEmpty(outputPath); + + // Assign each instrument configuration a stable integer index (cross-stack ids are ints). + // The map drives both the per-scan / run-default references and the OriginalId we stash so + // the reader can restore pwiz's real string ids. + var icIndexById = BuildInstrumentConfigIndex(msd); + + var spectra = TranslateSpectra(msd, icIndexById); + var chroms = TranslateChromatograms(msd); + var fileMetadata = BuildFileMetadata(msd, spectra, icIndexById); + + MzPeakWriter.Write(outputPath, spectra, fileMetadata, chroms); + } + + /// Maps each instrument configuration's (decoded) id to its 0-based position. + private static Dictionary BuildInstrumentConfigIndex(MSData msd) + { + var map = new Dictionary(StringComparer.Ordinal); + for (int i = 0; i < msd.InstrumentConfigurations.Count; i++) + { + var id = msd.InstrumentConfigurations[i].Id; + if (!string.IsNullOrEmpty(id)) map[id] = i; + } + return map; + } + + private static uint? ResolveInstrumentConfigRef(string? id, Dictionary icIndexById) + { + if (string.IsNullOrEmpty(id)) return null; + if (icIndexById.TryGetValue(id, out int idx)) return (uint)idx; + return TryParseId(id); // fall back to a numeric id (cross-stack files store ints directly) + } + + // ===== Spectrum translation ===== + + private static IReadOnlyList TranslateSpectra(MSData msd, Dictionary icIndexById) + { + var list = msd.Run.SpectrumList; + if (list is null || list.Count == 0) return Array.Empty(); + + var result = new List(list.Count); + for (int i = 0; i < list.Count; i++) + { + var s = list.GetSpectrum(i, getBinaryData: true); + result.Add(TranslateSpectrum(s, (ulong)i, icIndexById)); + } + return result; + } + + private static MzPeakWriter.SpectrumToWrite TranslateSpectrum(Spectrum s, ulong index, Dictionary icIndexById) + { + var intenArr = s.GetIntensityArray(); + // The "value" array is usually m/z, but UV/DAD spectra carry a wavelength array instead + // (no m/z). Use m/z when present, else the first non-intensity array-type binary array, and + // record its type/unit so the reader rebuilds the right array rather than assuming m/z. + var valueArr = s.GetMZArray() ?? FindNonIntensityValueArray(s); + var mz = valueArr is null ? Array.Empty() : valueArr.Data.ToArray(); + var intensity = NarrowToFloat(intenArr); + CVID valueTypeCvid = valueArr is null ? CVID.MS_m_z_array : ArrayTypeCvid(valueArr); + if (valueTypeCvid == CVID.CVID_Unknown) valueTypeCvid = CVID.MS_m_z_array; + string? valueArrayCurie = valueTypeCvid == CVID.MS_m_z_array ? null : CvLookup.CvTermInfo(valueTypeCvid).Id; + CVID valueUnitCvid = valueArr?.CvParam(valueTypeCvid).Units ?? CVID.CVID_Unknown; + string? valueArrayUnitCurie = (valueTypeCvid == CVID.MS_m_z_array || valueUnitCvid == CVID.CVID_Unknown) + ? null : CvLookup.CvTermInfo(valueUnitCvid).Id; + + Scan? scan0 = s.ScanList.Scans.Count > 0 ? s.ScanList.Scans[0] : null; + double time = ExtractScanStartTime(scan0); + int? msLevel = ExtractIntOrNull(s.Params, CVID.MS_ms_level); + bool isProfile = s.HasCVParam(CVID.MS_profile_spectrum); + bool hasRepresentation = isProfile || s.HasCVParam(CVID.MS_centroid_spectrum); + + // Spectrum-level scalars pulled from CV params. + int? scanPolarity = s.HasCVParam(CVID.MS_positive_scan) ? 1 + : s.HasCVParam(CVID.MS_negative_scan) ? -1 + : (int?)null; + double? basePeakMz = ExtractDoubleOrNull(s.Params, CVID.MS_base_peak_m_z); + double? basePeakIntensity = ExtractDoubleOrNull(s.Params, CVID.MS_base_peak_intensity); + double? tic = ExtractDoubleOrNull(s.Params, CVID.MS_total_ion_current); + double? lowMz = ExtractDoubleOrNull(s.Params, CVID.MS_lowest_observed_m_z); + double? highMz = ExtractDoubleOrNull(s.Params, CVID.MS_highest_observed_m_z); + + // Scan-level scalars. + string? filterString = scan0?.CvParam(CVID.MS_filter_string).Value; + if (string.IsNullOrEmpty(filterString)) filterString = null; + double? ionInjection = scan0 is null ? null : ExtractDoubleOrNull(scan0, CVID.MS_ion_injection_time); + long? preset = scan0 is null ? null : ExtractLongOrNull(scan0, CVID.MS_preset_scan_configuration); + (double? imValue, string? imTypeCurie) = ExtractIonMobility(scan0); + + // Instrument-config ref. mzPeak stores an integer index; map pwiz's string id to it. + uint? icRef = ResolveInstrumentConfigRef(scan0?.InstrumentConfiguration?.Id, icIndexById); + + // Scan windows split into parallel low/high arrays; per-window free-form params (beyond the + // lower/upper limits) ride a JSON sidecar so window-level annotations round-trip. + double?[]? scanWindowLowers = null; + double?[]? scanWindowUppers = null; + string? scanWindowParamsJson = null; + if (scan0 is not null && scan0.ScanWindows.Count > 0) + { + scanWindowLowers = new double?[scan0.ScanWindows.Count]; + scanWindowUppers = new double?[scan0.ScanWindows.Count]; + var windowParams = new List>(scan0.ScanWindows.Count); + for (int i = 0; i < scan0.ScanWindows.Count; i++) + { + scanWindowLowers[i] = ExtractDoubleOrNull(scan0.ScanWindows[i], CVID.MS_scan_window_lower_limit); + scanWindowUppers[i] = ExtractDoubleOrNull(scan0.ScanWindows[i], CVID.MS_scan_window_upper_limit); + windowParams.Add(ExtractWindowParams(scan0.ScanWindows[i])); + } + scanWindowParamsJson = ScanWindowParams.Serialize(windowParams); + } + + // Free-form params (those not handled by the typed columns above) flow + // through as CV-param lists so the round-trip preserves vendor-specific + // annotations the reader won't otherwise know about. + var spectrumParams = ExtractFreeFormParams(s.Params, SpectrumCvScalarBlacklist); + var scanParams = scan0 is null ? null : ExtractFreeFormParams(scan0, ScanCvScalarBlacklist); + + var precursors = TranslatePrecursors(s); + + // scanList combination method (MS:1000570 children) and referenceableParamGroup refs. + string? scanCombinationCurie = FindScanCombinationCurie(s.ScanList); + IReadOnlyList? paramGroupRefs = s.Params.ParamGroups.Count > 0 + ? s.Params.ParamGroups.Select(pg => pg.Id).ToList() + : null; + string? auxJson = AuxiliaryArrays.Serialize( + ExtractAuxiliaryArrays(s.BinaryDataArrays, s.IntegerDataArrays, valueArr)); + + // scanList scans beyond the first (combined ion-mobility spectra: one scan per mobility bin). + string? extraScansJson = ExtractExtraScans(s.ScanList); + + return new MzPeakWriter.SpectrumToWrite( + Index: index, + Id: s.Id, + Time: time, + MsLevel: msLevel, + IsProfile: isProfile, + Mz: mz, + Intensity: intensity, + ScanStartTime: scan0 is null ? (double?)null : ExtractDoubleOrNull(scan0, CVID.MS_scan_start_time), + FilterString: filterString, + InstrumentConfigurationRef: icRef, + IonInjectionTime: ionInjection, + ScanWindowLowerLimits: scanWindowLowers, + ScanWindowUpperLimits: scanWindowUppers, + ScanSpectrumRef: string.IsNullOrEmpty(scan0?.SpectrumId) ? null : scan0!.SpectrumId, + ScanWindowParamsJson: scanWindowParamsJson, + SpectrumParameters: spectrumParams, + ScanParameters: scanParams, + ScanPolarity: scanPolarity, + BasePeakMz: basePeakMz, + BasePeakIntensity: basePeakIntensity, + TotalIonCurrent: tic, + LowestObservedMz: lowMz, + HighestObservedMz: highMz, + ScanIonMobilityValue: imValue, + ScanIonMobilityTypeCurie: imTypeCurie, + PresetScanConfiguration: preset, + ScanCombinationCurie: scanCombinationCurie, + ParamGroupRefs: paramGroupRefs, + HasRepresentation: hasRepresentation, + ValueArrayCurie: valueArrayCurie, + ValueArrayUnitCurie: valueArrayUnitCurie, + AuxiliaryArraysJson: auxJson, + ExtraScansJson: extraScansJson, + Precursors: precursors); + } + + private static IReadOnlyList? TranslatePrecursors(Spectrum s) + { + if (s.Precursors.Count == 0) return null; + var result = new List(s.Precursors.Count); + foreach (var p in s.Precursors) + { + // Isolation window scalars. + double? targetMz = ExtractDoubleOrNull(p.IsolationWindow, CVID.MS_isolation_window_target_m_z); + double? lowerOff = ExtractDoubleOrNull(p.IsolationWindow, CVID.MS_isolation_window_lower_offset); + double? upperOff = ExtractDoubleOrNull(p.IsolationWindow, CVID.MS_isolation_window_upper_offset); + var isoParams = ExtractFreeFormParams(p.IsolationWindow, IsolationCvScalarBlacklist); + + // Activation: collision energy + the dissociation method CV term. + double? ce = ExtractDoubleOrNull(p.Activation, CVID.MS_collision_energy); + string? dissCurie = FindDissociationMethodCurie(p.Activation); + var actParams = ExtractFreeFormParams(p.Activation, ActivationCvScalarBlacklist); + + // Selected ion: take the first; multi-selected-ion-per-precursor + // (rare outside HRMS dedup) collapses to the first for now. + double? siMz = null, siIntensity = null; + long? siCharge = null; + IReadOnlyList? siParams = null; + if (p.SelectedIons.Count > 0) + { + var si = p.SelectedIons[0]; + siMz = ExtractDoubleOrNull(si, CVID.MS_selected_ion_m_z); + siIntensity = ExtractDoubleOrNull(si, CVID.MS_peak_intensity); + siCharge = ExtractLongOrNull(si, CVID.MS_charge_state); + siParams = ExtractFreeFormParams(si, SelectedIonCvScalarBlacklist); + } + + result.Add(new MzPeakWriter.PrecursorToWrite( + PrecursorId: string.IsNullOrEmpty(p.SpectrumId) ? null : p.SpectrumId, + IsolationTargetMz: targetMz, + IsolationLowerOffset: lowerOff, + IsolationUpperOffset: upperOff, + CollisionEnergy: ce, + DissociationMethodCurie: dissCurie, + SelectedIonMz: siMz, + SelectedIonPeakIntensity: siIntensity, + SelectedIonChargeState: siCharge, + IsolationWindowParameters: isoParams, + ActivationParameters: actParams, + SelectedIonParameters: siParams)); + } + return result; + } + + // ===== Chromatogram translation ===== + + private static IReadOnlyList TranslateChromatograms(MSData msd) + { + var list = msd.Run.ChromatogramList; + if (list is null || list.Count == 0) return Array.Empty(); + + var result = new List(list.Count); + for (int i = 0; i < list.Count; i++) + { + var c = list.GetChromatogram(i, getBinaryData: true); + var timeArr = c.GetTimeArray(); + var intenArr = c.GetIntensityArray(); + var time = timeArr is null ? Array.Empty() : timeArr.Data.ToArray(); + var intensity = NarrowToFloat(intenArr); + + // Chromatogram type CURIE: prefer the standard chromatogram types. + string? typeCurie = FindChromatogramTypeCurie(c.Params); + + // Preserve the time + intensity array units (both vary widely across vendors). + string? timeUnitCurie = null; + if (timeArr is not null) + { + var timeCv = timeArr.CvParam(CVID.MS_time_array); + if (timeCv.Units != CVID.CVID_Unknown) + timeUnitCurie = CvLookup.CvTermInfo(timeCv.Units).Id; + } + string? intensityUnitCurie = null; + if (intenArr is not null) + { + var intenCv = intenArr.CvParam(CVID.MS_intensity_array); + if (intenCv.Units != CVID.CVID_Unknown) + intensityUnitCurie = CvLookup.CvTermInfo(intenCv.Units).Id; + } + + result.Add(new MzPeakWriter.ChromatogramToWrite( + Index: (ulong)i, + Id: c.Id, + ChromatogramTypeCurie: typeCurie, + DataProcessingRef: c.DataProcessing?.Id, + Time: time, + Intensity: intensity, + TimeUnitCurie: timeUnitCurie, + IntensityUnitCurie: intensityUnitCurie, + Parameters: ExtractChromatogramParams(c.Params), + AuxiliaryArraysJson: AuxiliaryArrays.Serialize( + ExtractAuxiliaryArrays(c.BinaryDataArrays, c.IntegerDataArrays, timeArr)))); + } + return result; + } + + // ===== File-level metadata ===== + + private static MzPeakFileMetadata BuildFileMetadata(MSData msd, IReadOnlyList spectra, + Dictionary icIndexById) + { + // Source files + content CV params from MSData.FileDescription. + var contents = ToMzPeakCvParams(msd.FileDescription.FileContent); + var sourceFiles = new List(msd.FileDescription.SourceFiles.Count); + foreach (var sf in msd.FileDescription.SourceFiles) + { + sourceFiles.Add(new MzPeakSourceFile( + Id: sf.Id, + Name: sf.Name, + Location: sf.Location, + Parameters: ToMzPeakCvParams(sf))); + } + + // Instrument configurations: stable integer index for cross-stack readers, plus the real + // pwiz string id (OriginalId), the source/analyzer/detector component chain, and the + // controlling-software reference so the configuration round-trips in full. + var instrumentConfigs = new List(msd.InstrumentConfigurations.Count); + for (int i = 0; i < msd.InstrumentConfigurations.Count; i++) + { + var ic = msd.InstrumentConfigurations[i]; + instrumentConfigs.Add(new MzPeakInstrumentConfiguration( + Id: i, + Components: BuildComponents(ic.ComponentList), + SoftwareReference: ic.Software?.Id, + Parameters: ToMzPeakCvParams(ic), + OriginalId: string.IsNullOrEmpty(ic.Id) ? null : ic.Id, + ParamGroupRefs: ic.Params.ParamGroups.Count > 0 + ? ic.Params.ParamGroups.Select(pg => pg.Id).ToList() + : null)); + } + + var dpMethods = new List(msd.DataProcessings.Count); + foreach (var dp in msd.DataProcessings) + { + var methods = new List(dp.ProcessingMethods.Count); + foreach (var pm in dp.ProcessingMethods) + methods.Add(new ProcessingMethodInfo( + Order: pm.Order, + SoftwareReference: pm.Software?.Id, + Parameters: ToMzPeakCvParams(pm))); + dpMethods.Add(new DataProcessingMethod(Id: dp.Id, Methods: methods)); + } + + // referenceableParamGroupList: bundle id + its params so the reader can rebuild the shared + // groups (per-spectrum/scan references travel as a row-level column, populated elsewhere). + var paramGroups = new List(msd.ParamGroups.Count); + foreach (var pg in msd.ParamGroups) + paramGroups.Add(new ParamGroupInfo(Id: pg.Id, Parameters: ToMzPeakCvParams(pg))); + + var software = new List(msd.Software.Count); + foreach (var sw in msd.Software) + { + software.Add(new SoftwareInfo( + Id: sw.Id, + Version: sw.Version, + Parameters: ToMzPeakCvParams(sw))); + } + + var samples = new List(msd.Samples.Count); + foreach (var s in msd.Samples) + { + samples.Add(new SampleInfo( + Id: s.Id, + Name: s.Name, + Parameters: ToMzPeakCvParams(s))); + } + + uint? defaultInstrumentId = ResolveInstrumentConfigRef(msd.Run.DefaultInstrumentConfiguration?.Id, icIndexById); + var run = new RunInfo( + Id: msd.Run.Id, + DefaultDataProcessingId: null, + DefaultInstrumentId: defaultInstrumentId is uint u ? (int)u : null, + DefaultSourceFileId: msd.Run.DefaultSourceFile?.Id, + StartTime: string.IsNullOrEmpty(msd.Run.StartTimeStamp) ? null : msd.Run.StartTimeStamp, + Parameters: ToMzPeakCvParams(msd.Run)); + + long spectrumDataPointCount = 0; + foreach (var s in spectra) spectrumDataPointCount += s.Mz.Length; + + return new MzPeakFileMetadata( + FileDescription: new MzPeakFileDescription(Contents: contents, SourceFiles: sourceFiles), + InstrumentConfigurations: instrumentConfigs, + DataProcessingMethods: dpMethods, + Software: software, + Samples: samples, + Run: run, + SpectrumCount: spectra.Count, + SpectrumDataPointCount: spectrumDataPointCount, + DocumentId: string.IsNullOrEmpty(msd.Id) ? null : msd.Id, + ParamGroups: paramGroups.Count == 0 ? null : paramGroups); + } + + /// Translate a pwiz component chain into the mzPeak component records. + private static IReadOnlyList BuildComponents(ComponentList components) + { + if (components.Count == 0) return Array.Empty(); + var result = new List(components.Count); + foreach (var c in components) + result.Add(new ComponentInfo( + Type: ComponentTypeToString(c.Type), + Order: c.Order, + Parameters: ToMzPeakCvParams(c))); + return result; + } + + private static string ComponentTypeToString(ComponentType type) => type switch + { + ComponentType.Source => "source", + ComponentType.Analyzer => "analyzer", + ComponentType.Detector => "detector", + _ => "unknown", + }; + + // ===== Helpers ===== + + private static float[] NarrowToFloat(BinaryDataArray? source) + { + if (source is null) return Array.Empty(); + var arr = new float[source.Data.Count]; + for (int i = 0; i < source.Data.Count; i++) arr[i] = (float)source.Data[i]; + return arr; + } + + private static double ExtractScanStartTime(Scan? scan) + { + if (scan is null) return 0.0; + return ExtractDoubleOrNull(scan, CVID.MS_scan_start_time) ?? 0.0; + } + + private static double? ExtractDoubleOrNull(ParamContainer container, CVID cvid) + { + var p = container.CvParam(cvid); + if (p.Cvid == CVID.CVID_Unknown || string.IsNullOrEmpty(p.Value)) return null; + return p.ValueAs(); + } + + private static int? ExtractIntOrNull(ParamContainer container, CVID cvid) + { + var p = container.CvParam(cvid); + if (p.Cvid == CVID.CVID_Unknown || string.IsNullOrEmpty(p.Value)) return null; + return p.ValueAs(); + } + + private static long? ExtractLongOrNull(ParamContainer container, CVID cvid) + { + var p = container.CvParam(cvid); + if (p.Cvid == CVID.CVID_Unknown || string.IsNullOrEmpty(p.Value)) return null; + return p.ValueAs(); + } + + /// + /// Walk the scan's CV params looking for one of the ion-mobility value + /// terms (drift time, reverse drift time, FAIMS CV, …). Returns (value, + /// type CURIE) for the first match, or (null, null) when none is present. + /// + private static (double? value, string? typeCurie) ExtractIonMobility(Scan? scan) + { + if (scan is null) return (null, null); + foreach (var im in IonMobilityCvids) + { + var p = scan.CvParam(im); + if (p.Cvid != CVID.CVID_Unknown && !string.IsNullOrEmpty(p.Value)) + return (p.ValueAs(), CvLookup.CvTermInfo(im).Id); + } + return (null, null); + } + + private static string? FindDissociationMethodCurie(Activation activation) + { + foreach (var cvParam in activation.CVParams) + { + if (CvLookup.CvIsA(cvParam.Cvid, CVID.MS_dissociation_method)) + return CvLookup.CvTermInfo(cvParam.Cvid).Id; + } + return null; + } + + private static string? FindScanCombinationCurie(ScanList scanList) + { + foreach (var cvParam in scanList.CVParams) + if (CvLookup.CvIsA(cvParam.Cvid, CVID.MS_spectra_combination)) + return CvLookup.CvTermInfo(cvParam.Cvid).Id; + return null; + } + + /// + /// Chromatogram-level free-form params: every CV/user param except the chromatogram-type term + /// (which rides the typed type-CURIE column). Preserves polarity and any other annotations. + /// + private static IReadOnlyList? ExtractChromatogramParams(ParamContainer container) + { + var list = new List(); + foreach (var cv in container.CVParams) + { + if (CvLookup.CvIsA(cv.Cvid, CVID.MS_chromatogram_type)) continue; + list.Add(ToMzPeakCv(cv)); + } + foreach (var up in container.UserParams) + list.Add(new MzPeakReader.CvParam( + Name: up.Name, Accession: null, ValueString: up.Value, + ValueInteger: null, ValueFloat: null, ValueBoolean: null, + Unit: up.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(up.Units).Id, + Type: string.IsNullOrEmpty(up.Type) ? null : up.Type)); + return list.Count == 0 ? null : list; + } + + private static readonly HashSet ArrayEncodingCvids = new() + { + CVID.MS_32_bit_float, CVID.MS_64_bit_float, CVID.MS_32_bit_integer, CVID.MS_64_bit_integer, + CVID.MS_no_compression, CVID.MS_zlib_compression, + CVID.MS_MS_Numpress_linear_prediction_compression, + CVID.MS_MS_Numpress_positive_integer_compression, + CVID.MS_MS_Numpress_short_logged_float_compression, + CVID.MS_MS_Numpress_linear_prediction_compression_followed_by_zlib_compression, + CVID.MS_MS_Numpress_positive_integer_compression_followed_by_zlib_compression, + CVID.MS_MS_Numpress_short_logged_float_compression_followed_by_zlib_compression, + }; + + /// + /// Capture every binary/integer data array beyond the canonical value+intensity pair + /// ( = the m/z / wavelength / time array; intensity excluded by CV) + /// as records — ion-mobility arrays, "ms level" non-standard + /// arrays, resolution/baseline/SN arrays, etc. Binary-encoding terms (precision, compression) + /// are dropped; the array-type / name / unit params are kept so the reader can rebuild them. + /// + private static IReadOnlyList? ExtractAuxiliaryArrays( + IReadOnlyList binaryArrays, IReadOnlyList integerArrays, BinaryDataArray? valueArray) + { + var result = new List(); + foreach (var arr in binaryArrays) + { + if (ReferenceEquals(arr, valueArray) || arr.HasCVParam(CVID.MS_intensity_array)) continue; + result.Add(new AuxiliaryArrayData(AuxArrayParams(arr), IsInteger: false, DoubleValues: arr.Data.ToArray(), IntValues: null)); + } + foreach (var arr in integerArrays) + result.Add(new AuxiliaryArrayData(AuxArrayParams(arr), IsInteger: true, DoubleValues: null, IntValues: arr.Data.ToArray())); + return result.Count == 0 ? null : result; + } + + /// The array-type CV term (child of MS:1000513 binary data array) identifying an array. + private static CVID ArrayTypeCvid(ParamContainer arr) + { + foreach (var cv in arr.CVParams) + if (CvLookup.CvIsA(cv.Cvid, CVID.MS_binary_data_array)) + return cv.Cvid; + return CVID.CVID_Unknown; + } + + /// First non-intensity array-type binary array (the value array when there is no m/z, e.g. UV wavelength). + private static BinaryDataArray? FindNonIntensityValueArray(Spectrum s) + { + foreach (var arr in s.BinaryDataArrays) + { + if (arr.HasCVParam(CVID.MS_intensity_array)) continue; + if (ArrayTypeCvid(arr) != CVID.CVID_Unknown) return arr; + } + return null; + } + + /// Serialize scanList scans beyond scan[0] (their full params + scan windows) to JSON. + private static string? ExtractExtraScans(ScanList scanList) + { + if (scanList.Scans.Count <= 1) return null; + var extras = new List(scanList.Scans.Count - 1); + for (int i = 1; i < scanList.Scans.Count; i++) + { + var scan = scanList.Scans[i]; + var windows = new List>(scan.ScanWindows.Count); + foreach (var w in scan.ScanWindows) windows.Add(ToMzPeakCvParams(w)); + extras.Add(new ExtraScanData(ToMzPeakCvParams(scan), windows, + SpectrumId: string.IsNullOrEmpty(scan.SpectrumId) ? null : scan.SpectrumId)); + } + return ExtraScans.Serialize(extras); + } + + /// Scan-window params beyond the typed lower/upper limits (e.g. Agilent "centroided min/max"). + private static IReadOnlyList ExtractWindowParams(ParamContainer window) + { + var list = new List(); + foreach (var cv in window.CVParams) + { + if (cv.Cvid is CVID.MS_scan_window_lower_limit or CVID.MS_scan_window_upper_limit) continue; + list.Add(new MzPeakCvParam( + Name: CvLookup.CvTermInfo(cv.Cvid).Name, + Accession: CvLookup.CvTermInfo(cv.Cvid).Id, + Value: CoerceJsonScalar(cv.Value), + Unit: cv.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(cv.Units).Id)); + } + foreach (var up in window.UserParams) + list.Add(new MzPeakCvParam( + Name: up.Name, Accession: null, Value: up.Value, + Unit: up.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(up.Units).Id)); + return list; + } + + private static IReadOnlyList AuxArrayParams(ParamContainer arr) + { + var list = new List(); + foreach (var cv in arr.CVParams) + { + if (ArrayEncodingCvids.Contains(cv.Cvid)) continue; + list.Add(new MzPeakCvParam( + Name: CvLookup.CvTermInfo(cv.Cvid).Name, + Accession: CvLookup.CvTermInfo(cv.Cvid).Id, + Value: CoerceJsonScalar(cv.Value), + Unit: cv.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(cv.Units).Id)); + } + foreach (var up in arr.UserParams) + list.Add(new MzPeakCvParam( + Name: up.Name, Accession: null, Value: up.Value, + Unit: up.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(up.Units).Id)); + return list; + } + + private static string? FindChromatogramTypeCurie(ParamContainer container) + { + foreach (var cvParam in container.CVParams) + { + // Chromatogram-type terms (TIC / SIC / BPC / …) are children of MS:1000626 + // "chromatogram type", NOT MS:1000625 "chromatogram" — walking the wrong root + // silently dropped the type on round-trip. + if (CvLookup.CvIsA(cvParam.Cvid, CVID.MS_chromatogram_type)) + return CvLookup.CvTermInfo(cvParam.Cvid).Id; + } + return null; + } + + /// + /// Extract every CV/User param NOT already represented by one of the typed + /// scalar columns. Keeps round-trip lossless for vendor-specific terms. + /// + private static IReadOnlyList? ExtractFreeFormParams(ParamContainer container, HashSet blacklist) + { + var list = new List(); + foreach (var cv in container.CVParams) + { + if (blacklist.Contains(cv.Cvid)) continue; + // Also skip polarity / representation CVs that were folded into + // scalar columns; they have no value text and would re-emit empty. + if (PolarityRepresentationCvids.Contains(cv.Cvid)) continue; + list.Add(ToMzPeakCv(cv)); + } + foreach (var up in container.UserParams) + { + // Free-form user params survive as CV-less name/value pairs, keeping their xsd type hint. + list.Add(new MzPeakReader.CvParam( + Name: up.Name, + Accession: null, + ValueString: up.Value, + ValueInteger: null, + ValueFloat: null, + ValueBoolean: null, + Unit: up.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(up.Units).Id, + Type: string.IsNullOrEmpty(up.Type) ? null : up.Type)); + } + return list.Count == 0 ? null : list; + } + + private static IReadOnlyList ToMzPeakCvParams(ParamContainer container) + { + var list = new List(container.CVParams.Count + container.UserParams.Count); + foreach (var cv in container.CVParams) + { + list.Add(new MzPeakCvParam( + Name: CvLookup.CvTermInfo(cv.Cvid).Name, + Accession: CvLookup.CvTermInfo(cv.Cvid).Id, + Value: CoerceJsonScalar(cv.Value), + Unit: cv.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(cv.Units).Id)); + } + foreach (var up in container.UserParams) + { + list.Add(new MzPeakCvParam( + Name: up.Name, + Accession: null, + Value: up.Value, + Unit: up.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(up.Units).Id)); + } + return list; + } + + private static MzPeakReader.CvParam ToMzPeakCv(CVParam cv) + { + // We only know the value is a string here — pwiz CV params don't carry + // typed values, just an XSD-style text representation. Always pack as + // ValueString; readers needing typed access can parse on demand. + return new MzPeakReader.CvParam( + Name: CvLookup.CvTermInfo(cv.Cvid).Name, + Accession: CvLookup.CvTermInfo(cv.Cvid).Id, + ValueString: cv.Value, + ValueInteger: null, + ValueFloat: null, + ValueBoolean: null, + Unit: cv.Units == CVID.CVID_Unknown ? null : CvLookup.CvTermInfo(cv.Units).Id); + } + + private static object? CoerceJsonScalar(string? value) + { + if (string.IsNullOrEmpty(value)) return null; + // Persist as the most specific JSON scalar type we can recognise so + // round-trip readers can parse without extra interpretation. + if (long.TryParse(value, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var l)) return l; + if (double.TryParse(value, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var d)) return d; + if (string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)) return true; + if (string.Equals(value, "false", StringComparison.OrdinalIgnoreCase)) return false; + return value; + } + + private static uint? TryParseId(string? id) + { + if (string.IsNullOrEmpty(id)) return null; + if (uint.TryParse(id, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var v)) + return v; + return null; + } + + // ===== CV blacklists ===== + // + // CV terms we extract into typed scalar columns are filtered out of the free-form parameter + // lists to avoid emitting the same value twice. BUT only *unitless* terms are blacklisted: + // every unit-bearing scalar's unit varies by vendor — present, absent, or different (e.g. + // total ion current is unitless in some vendors but MS_number_of_detector_counts in others; + // base peak m/z is unitless in Waters but MS_m_z elsewhere; ion mobility is ms vs Vs/cm²) — + // and the typed columns can't carry the unit. So unit-bearing scalars are left OUT of the + // blacklist: they ride the free-form params (which preserve the exact unit, including its + // absence) while their value also lands in the typed column for cross-stack consumers. On read + // the typed column seeds a synthesized-unit fallback that the free-form param then overrides, + // so cross-stack files (no free-form params) still get a sensible default. + + private static readonly HashSet SpectrumCvScalarBlacklist = new() + { + CVID.MS_ms_level, // unitless + }; + + private static readonly HashSet ScanCvScalarBlacklist = new() + { + CVID.MS_filter_string, // unitless (string) + CVID.MS_preset_scan_configuration, // unitless + }; + + private static readonly HashSet IsolationCvScalarBlacklist = new(); + + private static readonly HashSet ActivationCvScalarBlacklist = new(); + + private static readonly HashSet SelectedIonCvScalarBlacklist = new() + { + CVID.MS_charge_state, // unitless + }; + + private static readonly HashSet PolarityRepresentationCvids = new() + { + CVID.MS_positive_scan, + CVID.MS_negative_scan, + CVID.MS_profile_spectrum, + CVID.MS_centroid_spectrum, + }; + + private static readonly CVID[] IonMobilityCvids = new[] + { + CVID.MS_ion_mobility_drift_time, + CVID.MS_inverse_reduced_ion_mobility, + CVID.MS_FAIMS_compensation_voltage, + }; +} diff --git a/pwiz-sharp/pwiz/src/MsData/MzPeakReaderAdapter.cs b/pwiz-sharp/pwiz/src/MsData/MzPeakReaderAdapter.cs new file mode 100644 index 00000000000..f0247c8f757 --- /dev/null +++ b/pwiz-sharp/pwiz/src/MsData/MzPeakReaderAdapter.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.Common.Params; +using Pwiz.Data.MsData.Instruments; +using Pwiz.Data.MsData.MzPeak; +using Pwiz.Data.MsData.Processing; +using Pwiz.Data.MsData.Samples; +using Pwiz.Data.MsData.Sources; +// MzPeak's file-level metadata records share a lot of names with pwiz's +// MSData model (SourceFile, InstrumentConfiguration, FileDescription, …). +// Alias the mzPeak side so the translation methods can mention both. +using MzPeakFileMetadata = Pwiz.Data.MsData.MzPeak.FileMetadata; +using MzPeakSourceFile = Pwiz.Data.MsData.MzPeak.SourceFile; +using MzPeakInstrumentConfiguration = Pwiz.Data.MsData.MzPeak.InstrumentConfiguration; + +namespace Pwiz.Data.MsData.Readers; + +/// Identifies and reads mzPeak files (mass-spec data packed as Parquet tables inside a ZIP container). +/// +/// A mzPeak file is a ZIP archive holding spectra_metadata.parquet, +/// spectra_data.parquet, optional spectra_peaks.parquet, and the +/// chromatogram parquets, plus a mzpeak_index.json manifest. We +/// recognize it by extension first; without an extension hit, fall back to ZIP +/// magic + presence of the manifest entry. +/// +public sealed class MzPeakReaderAdapter : IReader +{ + /// + public string TypeName => "mzPeak"; + + /// + public CVID CvType => CVID.MS_mzPeak_format; + + /// + public IReadOnlyList FileExtensions { get; } = new[] { ".mzpeak", ".mzPeak" }; + + // ZIP local-file-header magic. Every ZIP archive starts with these 4 bytes + // (PK\3\4). Not every ZIP is mzPeak though — the secondary check is the + // presence of the mzpeak_index.json manifest entry. + private static readonly byte[] ZipMagic = { 0x50, 0x4B, 0x03, 0x04 }; + + /// + public CVID Identify(string filename, string? head) + { + ArgumentNullException.ThrowIfNull(filename); + + foreach (var ext in FileExtensions) + if (filename.EndsWith(ext, StringComparison.OrdinalIgnoreCase)) + return CvType; + + if (head is null || head.Length < ZipMagic.Length) return CVID.CVID_Unknown; + for (int i = 0; i < ZipMagic.Length; i++) + if ((byte)head[i] != ZipMagic[i]) return CVID.CVID_Unknown; + + try + { + using var archive = System.IO.Compression.ZipFile.OpenRead(filename); + if (archive.GetEntry("mzpeak_index.json") != null) return CvType; + } + catch + { + // Malformed ZIP or no permission to open — fall through to unknown. + } + return CVID.CVID_Unknown; + } + + /// + public void Read(string filename, MSData result, ReaderConfig? config = null) + { + ArgumentNullException.ThrowIfNull(filename); + ArgumentNullException.ThrowIfNull(result); + + // Open the reader eagerly — it extracts the archive to a scratch dir + // and loads every metadata column into memory at construction. The + // SpectrumList / ChromatogramList we install below share ownership of + // the reader; the first list disposed cleans up the scratch dir, and + // MzPeakReader.Dispose is a no-op on a second call. + var reader = new MzPeakReader(filename); + try + { + var fm = reader.FileMetadata; + + if (!string.IsNullOrEmpty(fm.DocumentId)) result.Id = fm.DocumentId!; + + TranslateFileDescription(fm, result); + // Software first: instrument configurations and dataProcessing methods reference it by id. + TranslateSoftware(fm, result); + TranslateParamGroups(fm, result); + TranslateInstrumentConfigurations(fm, result); + TranslateDataProcessings(fm, result); + TranslateSamples(fm, result); + TranslateRun(fm, result); + + var dp = MSDataFile.FillInCommonMetadata(filename, result); + + // Ownership: spectrum list owns the reader, chromatogram list rides + // along. If there's no chromatogram list (count == 0) we still + // install one because callers iterate Run.ChromatogramList.Count. + result.Run.SpectrumList = new SpectrumList_MzPeak(reader, dp, ownsReader: true, result.ParamGroups); + result.Run.ChromatogramList = new ChromatogramList_MzPeak(reader, dp, ownsReader: false); + reader = null!; // ownership transferred + } + finally + { + reader?.Dispose(); + } + } + + // ===== File-metadata translation ===== + + private static void TranslateFileDescription(MzPeakFileMetadata fm, MSData result) + { + // FileContent: bag of CV params describing the kinds of spectra inside. + ApplyCvParams(result.FileDescription.FileContent, fm.FileDescription.Contents); + + foreach (MzPeakSourceFile sf in fm.FileDescription.SourceFiles) + { + var pwizSf = new Sources.SourceFile( + id: sf.Id ?? string.Empty, + name: sf.Name ?? string.Empty, + location: sf.Location ?? string.Empty); + ApplyCvParams(pwizSf, sf.Parameters); + result.FileDescription.SourceFiles.Add(pwizSf); + } + } + + private static void TranslateParamGroups(MzPeakFileMetadata fm, MSData result) + { + if (fm.ParamGroups is null) return; + foreach (var pg in fm.ParamGroups) + { + var pwizPg = new Pwiz.Data.Common.Params.ParamGroup(pg.Id ?? string.Empty); + ApplyCvParams(pwizPg, pg.Parameters); + result.ParamGroups.Add(pwizPg); + } + } + + private static void TranslateInstrumentConfigurations(MzPeakFileMetadata fm, MSData result) + { + foreach (MzPeakInstrumentConfiguration ic in fm.InstrumentConfigurations) + { + // Restore pwiz's real string id when we stashed it; otherwise fall back to the + // cross-stack integer index (files written by other stacks only carry the int). + string id = string.IsNullOrEmpty(ic.OriginalId) + ? ic.Id.ToString(System.Globalization.CultureInfo.InvariantCulture) + : ic.OriginalId!; + var pwizIc = new Instruments.InstrumentConfiguration(id); + ApplyCvParams(pwizIc, ic.Parameters); + + foreach (var comp in ic.Components ?? System.Linq.Enumerable.Empty()) + { + var pwizComp = new Instruments.Component(ComponentTypeFromString(comp.Type), comp.Order); + ApplyCvParams(pwizComp, comp.Parameters); + pwizIc.ComponentList.Add(pwizComp); + } + + if (!string.IsNullOrEmpty(ic.SoftwareReference)) + pwizIc.Software = result.Software.FirstOrDefault(s => s.Id == ic.SoftwareReference); + + if (ic.ParamGroupRefs is not null) + foreach (var refId in ic.ParamGroupRefs) + { + var pg = result.ParamGroups.FirstOrDefault(g => g.Id == refId); + if (pg is null) continue; + pwizIc.Params.ParamGroups.Add(pg); + // Drop any direct param a referenced group already provides (avoid duplication). + var provided = pg.CVParams.Select(cv => cv.Cvid).ToHashSet(); + pwizIc.CVParams.RemoveAll(cv => provided.Contains(cv.Cvid)); + } + + result.InstrumentConfigurations.Add(pwizIc); + } + } + + private static void TranslateDataProcessings(MzPeakFileMetadata fm, MSData result) + { + foreach (var dp in fm.DataProcessingMethods) + { + var pwizDp = new Processing.DataProcessing(dp.Id ?? string.Empty); + foreach (var m in dp.Methods ?? System.Linq.Enumerable.Empty()) + { + var pwizPm = new Processing.ProcessingMethod { Order = m.Order }; + if (!string.IsNullOrEmpty(m.SoftwareReference)) + pwizPm.Software = result.Software.FirstOrDefault(s => s.Id == m.SoftwareReference); + ApplyCvParams(pwizPm, m.Parameters); + pwizDp.ProcessingMethods.Add(pwizPm); + } + result.DataProcessings.Add(pwizDp); + } + } + + private static Instruments.ComponentType ComponentTypeFromString(string? type) => type switch + { + "source" => Instruments.ComponentType.Source, + "analyzer" => Instruments.ComponentType.Analyzer, + "detector" => Instruments.ComponentType.Detector, + _ => Instruments.ComponentType.Unknown, + }; + + private static void TranslateSoftware(MzPeakFileMetadata fm, MSData result) + { + foreach (var sw in fm.Software) + { + var pwizSw = new Software(sw.Id ?? string.Empty) + { + Version = sw.Version ?? string.Empty, + }; + ApplyCvParams(pwizSw, sw.Parameters); + result.Software.Add(pwizSw); + } + } + + private static void TranslateSamples(MzPeakFileMetadata fm, MSData result) + { + foreach (var s in fm.Samples) + { + var pwizS = new Sample(s.Id ?? string.Empty, s.Name ?? string.Empty); + ApplyCvParams(pwizS, s.Parameters); + result.Samples.Add(pwizS); + } + } + + private static void TranslateRun(MzPeakFileMetadata fm, MSData result) + { + if (!string.IsNullOrEmpty(fm.Run.Id)) result.Run.Id = fm.Run.Id!; + + if (!string.IsNullOrEmpty(fm.Run.StartTime)) result.Run.StartTimeStamp = fm.Run.StartTime!; + + if (!string.IsNullOrEmpty(fm.Run.DefaultSourceFileId)) + result.Run.DefaultSourceFile = result.FileDescription.SourceFiles + .FirstOrDefault(s => s.Id == fm.Run.DefaultSourceFileId); + + // DefaultInstrument: the run stores the configuration's integer index (null when the run had + // no default); the configurations were added in that same order, so resolve by position. + if (fm.Run.DefaultInstrumentId is int instrIdx && instrIdx >= 0 && instrIdx < result.InstrumentConfigurations.Count) + result.Run.DefaultInstrumentConfiguration = result.InstrumentConfigurations[instrIdx]; + + ApplyCvParams(result.Run, fm.Run.Parameters); + } + + // ===== Shared CV param helper ===== + + /// Apply a list of mzPeak-shaped CV params to a pwiz . + /// Tolerates a null list (a foreign writer may omit the field entirely). + private static void ApplyCvParams(ParamContainer target, IReadOnlyList? src) + { + if (src is null) return; + foreach (var p in src) + { + var cvid = CvidFromCurie(p.Accession); + var unitCvid = CvidFromCurie(p.Unit); + string value = ValueToString(p.Value); + + if (cvid != CVID.CVID_Unknown) + { + target.Set(cvid, value, unitCvid); + } + else + { + target.UserParams.Add(new Pwiz.Data.Common.Params.UserParam( + p.Name ?? string.Empty, value, type: string.Empty, units: unitCvid)); + } + } + } + + private static string ValueToString(object? value) => value switch + { + null => string.Empty, + string s => s, + bool b => b ? "true" : "false", + long l => l.ToString(System.Globalization.CultureInfo.InvariantCulture), + int i => i.ToString(System.Globalization.CultureInfo.InvariantCulture), + double d => d.ToString("R", System.Globalization.CultureInfo.InvariantCulture), + float f => f.ToString("R", System.Globalization.CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty, + }; + + private static CVID CvidFromCurie(string? curie) + { + if (string.IsNullOrEmpty(curie)) return CVID.CVID_Unknown; + return CvLookup.CvTermInfo(curie).Cvid; + } +} diff --git a/pwiz-sharp/pwiz/src/MsData/WriteConfig.cs b/pwiz-sharp/pwiz/src/MsData/WriteConfig.cs index f2f67fd3660..e524bde8e85 100644 --- a/pwiz-sharp/pwiz/src/MsData/WriteConfig.cs +++ b/pwiz-sharp/pwiz/src/MsData/WriteConfig.cs @@ -19,6 +19,8 @@ public enum WriteFormat Mz5, /// mzMLb HDF5 format (unimplemented). MzMLb, + /// mzPeak Parquet-archive format. + MzPeak, /// pwiz internal text format (unimplemented). Text, /// Legacy MS1 text format. diff --git a/pwiz-sharp/pwiz/src/TestHarness/ReaderTestConfig.cs b/pwiz-sharp/pwiz/src/TestHarness/ReaderTestConfig.cs index 0715bcc5149..5ccddee1ec9 100644 --- a/pwiz-sharp/pwiz/src/TestHarness/ReaderTestConfig.cs +++ b/pwiz-sharp/pwiz/src/TestHarness/ReaderTestConfig.cs @@ -132,6 +132,15 @@ public sealed record ReaderTestConfig /// public bool TestMzmlbRoundTrip { get; set; } = true; + /// + /// When true, the harness writes the in-memory MSData to mzPeak (Parquet-backed), + /// reads it back through MzPeakReaderAdapter, and diffs against the original + /// at the spectrum-data level. Same shape as the mzMLb check; mzPeak's column-typed + /// schema is the format under active development in this branch, so this round-trip + /// is the canary for translation regressions. + /// + public bool TestMzPeakRoundTrip { get; set; } = true; + /// /// When true, the HDF5-backed round-trip () still /// runs when the test process is being diff --git a/pwiz-sharp/pwiz/src/TestHarness/VendorReaderTestHarness.cs b/pwiz-sharp/pwiz/src/TestHarness/VendorReaderTestHarness.cs index 5d04cc4759f..aff0a991bed 100644 --- a/pwiz-sharp/pwiz/src/TestHarness/VendorReaderTestHarness.cs +++ b/pwiz-sharp/pwiz/src/TestHarness/VendorReaderTestHarness.cs @@ -440,6 +440,15 @@ private static void ReadAndDiff(IReader reader, string rawPath, string rootPath, RunMzmlbRoundTrip(msd, config, encoderConfig, diffPrecision: 1.0); } + // 9. mzPeak round-trip: Parquet-archive format under active development in this + // branch. Same shape as the mzMLb round-trip — write the in-memory MSData, read + // it back through the adapter, diff at the spectrum-data level. Tolerance matches + // mzMLb (1.0 abs) because mzPeak narrows intensity to float32 too. + if (config.TestMzPeakRoundTrip && msd.Run.SpectrumList is not null) + { + RunMzPeakRoundTrip(msd, diffPrecision: 1.0); + } + // mz5 round-trip removed — there's no in-process mz5 writer (the C# port is // read-only), and the previous workaround (write mzML, shell out to cpp // msconvert.exe --mz5, read the mz5 back) made pwiz-sharp's CI results depend @@ -484,7 +493,7 @@ private static void RunMzmlbRoundTrip( new Pwiz.Data.MsData.MzMlb.MzMlbWriter(encoderConfig).Write(msd, tmp); var roundtripped = new MSData(); new Pwiz.Data.MsData.Readers.MzMlbReaderAdapter().Read(tmp, roundtripped); - string report = MSDataDiff.DescribeSpectraDataOnly(msd, roundtripped, diffPrecision); + string report = DescribeRoundTripFull(msd, roundtripped, diffPrecision); if (report.Length > 0) throw new InvalidOperationException("mzMLb round-trip diff:\n" + report); } @@ -494,6 +503,85 @@ private static void RunMzmlbRoundTrip( } } + /// + /// Full-metadata round-trip diff for the lossless binary formats (mzMLb, mzPeak), which are + /// meant to be mzML-complete. Unlike this + /// compares the whole document (file description, instrument configurations, data processing, + /// per-spectrum scan/precursor metadata, chromatograms). is the + /// absolute tolerance that accommodates the format's float32 intensity narrowing. + /// + private static string DescribeRoundTripFull(MSData original, MSData roundtripped, double precision) + => MSDataDiff.DescribeRoundTrip(original, roundtripped, precision); + + /// + /// One mzPeak write+read cycle. Writes via + /// to a temp .mzpeak path, + /// reads it back with , + /// and diffs the spectrum data at the requested precision. + /// + private static void RunMzPeakRoundTrip(MSData msd, double diffPrecision) + { + string tmp = Path.Combine( + Path.GetTempPath(), + $"harness-mzpeak-{Guid.NewGuid():N}.mzpeak"); + try + { + Pwiz.Data.MsData.MzPeak.WriterMzPeak.Write(msd, tmp); + var roundtripped = new MSData(); + new Pwiz.Data.MsData.Readers.MzPeakReaderAdapter().Read(tmp, roundtripped); + string report = DescribeRoundTripFull(msd, roundtripped, diffPrecision); + if (report.Length > 0) + throw new InvalidOperationException("mzPeak round-trip diff:\n" + report); + + // Cross-stack verify: when MZPEAK_VERIFY_NET_SCRIPT points at a + // dotnet-script .csx that opens the file with mzPeak.NET, invoke + // it as a subprocess and fail this round-trip on non-zero exit. + // Gated on the env var so the in-process round-trip can still run + // on machines without dotnet-script installed. + VerifyMzPeakDotNet(tmp); + } + finally + { + try { File.Delete(tmp); } catch { } + } + } + + private static void VerifyMzPeakDotNet(string mzpeakPath) + { + string? scriptPath = Environment.GetEnvironmentVariable("MZPEAK_VERIFY_NET_SCRIPT"); + if (string.IsNullOrEmpty(scriptPath)) return; + + var psi = new System.Diagnostics.ProcessStartInfo + { + FileName = "dotnet", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + psi.ArgumentList.Add("script"); + psi.ArgumentList.Add(scriptPath); + psi.ArgumentList.Add("--"); + psi.ArgumentList.Add(mzpeakPath); + + using var proc = System.Diagnostics.Process.Start(psi) + ?? throw new InvalidOperationException("could not start dotnet script process"); + string stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + if (proc.ExitCode != 0) + { + // On failure, copy the .mzpeak aside so we can re-run the verifier + // manually. The original path lives under %TEMP% and is deleted by + // the surrounding finally, so without this we'd lose the artifact. + string saved = Path.Combine(Path.GetTempPath(), $"mzpeak-failed-{Path.GetFileName(mzpeakPath)}"); + try { File.Copy(mzpeakPath, saved, overwrite: true); } catch { } + throw new InvalidOperationException( + $"mzPeak.NET cross-stack verify failed (exit {proc.ExitCode}):\n" + + $"saved bad file to: {saved}\n" + + stdout + stderr); + } + } + /// /// True when the current process is being instrumented by a .NET coverage / profiler /// tool (dotCover, OpenCover, coverlet collector, etc.). Detected via the standard @@ -621,7 +709,12 @@ private static MSData BuildMgfFilteredCopy(MSData original) { var spec = sl.GetSpectrum(i, getBinaryData: true); if (!MgfSerializer.IsMgfWritable(spec)) continue; - spec.Index = filtered.Spectra.Count; + // NOTE: do NOT mutate spec.Index here. Some source lists (e.g. the SpectrumListSimple + // produced by the IndexRange path) hand back the SAME spectrum instances the original + // MSData holds, so re-indexing them would corrupt the original's indices — which then + // surfaces in the later full-metadata mzMLb/mzPeak round-trips as a spurious + // "index: A vs B" diff. MGF doesn't use Spectrum.Index, and the MGF round-trip diff is + // position-based, so leaving the original index intact is correct. filtered.Spectra.Add(spec); } copy.Run.SpectrumList = filtered; diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MsData.Tests.csproj b/pwiz-sharp/pwiz/test/MsData.Tests/MsData.Tests.csproj index 96db62f881a..cfdc7e9decc 100644 --- a/pwiz-sharp/pwiz/test/MsData.Tests/MsData.Tests.csproj +++ b/pwiz-sharp/pwiz/test/MsData.Tests/MsData.Tests.csproj @@ -32,4 +32,15 @@ + + + + PreserveNewest + mzpeak_crossstack\%(Filename)%(Extension) + + + diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_has_uv.mzpeak b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_has_uv.mzpeak new file mode 100644 index 00000000000..4b7d10c66dc Binary files /dev/null and b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_has_uv.mzpeak differ diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_small.mzpeak b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_small.mzpeak new file mode 100644 index 00000000000..e46bb6a0a63 Binary files /dev/null and b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_small.mzpeak differ diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_small_chunked.mzpeak b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_small_chunked.mzpeak new file mode 100644 index 00000000000..e9f1d7cb549 Binary files /dev/null and b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackData/mzpeaknet_small_chunked.mzpeak differ diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackReadTests.cs b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackReadTests.cs new file mode 100644 index 00000000000..11bc7d54422 --- /dev/null +++ b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/CrossStackReadTests.cs @@ -0,0 +1,158 @@ +using System; +using System.IO; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Pwiz.Data.MsData.MzPeak; + +namespace Pwiz.Data.MsData.Tests; + +/// +/// Reads mzPeak files written by the independent mzPeak.NET (Apache-Arrow) stack, not by our own +/// writer. These exercise the cross-stack robustness the reader needs beyond round-tripping its own +/// output: physical-type/width divergences (uint8 ms_level vs our signed int8, float32 scalars vs +/// double, int32/uint32 small-ints vs our int64), row-group discovery from Parquet statistics when +/// the point_row_group_ranges KV is absent, float32 value arrays in the point layer, and the +/// separate wavelength_spectra_* entries mzPeak.NET uses for UV/DAD spectra. +/// +/// Expected values were captured from mzPeak.NET's own reader over the same fixtures, so a divergence +/// here means our read of a foreign file disagrees with the writer's read of it. +/// +[TestClass] +public class CrossStackReadTests +{ + private static string FixtureDir => + Path.Combine(AppContext.BaseDirectory, "mzpeak_crossstack"); + + private static string Small => Path.Combine(FixtureDir, "mzpeaknet_small.mzpeak"); + private static string HasUv => Path.Combine(FixtureDir, "mzpeaknet_has_uv.mzpeak"); + private static string SmallChunked => Path.Combine(FixtureDir, "mzpeaknet_small_chunked.mzpeak"); + + [TestMethod] + public void Small_ProfileAndCentroidSpectra_ReadWithCorrectValues() + { + using var reader = new MzPeakReader(Small); + + Assert.AreEqual(48, reader.SpectrumCount); + + // Spectrum 0: profile, m/z array stored inline. Values from mzPeak.NET's reader. + var s0 = reader.GetSpectrumDescription(0); + Assert.IsTrue(s0.IsProfile, "spectrum 0 should be profile"); + Assert.AreEqual(1, s0.MsLevel, "ms_level is written uint8 by mzPeak.NET; must read back as 1"); + // number_of_data_points is written UInt64 by mzPeak.NET; guard the unsigned->long widening + // (a too-narrow conversion ladder would silently drop this to null). + Assert.AreEqual(13589L, s0.NumberOfDataPoints); + var d0 = reader.GetSpectrumData(0); + Assert.IsNotNull(d0); + Assert.AreEqual(13589, d0!.Mz.Length); + Assert.AreEqual(202.60657, d0.Mz[0], 1e-4); + Assert.AreEqual(202.60682, d0.Mz[1], 1e-4); + Assert.AreEqual(0.0, d0.Intensity[0], 1e-3); + Assert.AreEqual(1938.117, d0.Intensity[1], 1e-2); + + // Spectrum 2: centroid, data stored in the supplementary peaks layer. + var s2 = reader.GetSpectrumDescription(2); + Assert.IsTrue(s2.IsCentroid, "spectrum 2 should be centroid"); + var d2 = reader.GetSpectrumData(2); + Assert.IsNotNull(d2); + Assert.AreEqual(485, d2!.Mz.Length); + Assert.AreEqual(231.38884, d2.Mz[0], 1e-4); + } + + [TestMethod] + public void HasUv_WavelengthSpectra_AppendedAfterMsSpectraWithCorrectValues() + { + using var reader = new MzPeakReader(HasUv); + + // 212 MS spectra + 520 UV/DAD wavelength spectra (separate wavelength_spectra_* entries). + Assert.AreEqual(732, reader.SpectrumCount); + + // First wavelength spectrum is at global index 212 (right after the MS spectra). + var uv = reader.GetSpectrumDescription(212); + Assert.AreEqual("merged=212 row=0", uv.Id); + Assert.IsTrue(uv.IsProfile); + // The value array is wavelength (nm), not m/z. + Assert.AreEqual("MS:1000617", uv.ValueArrayCurie); + Assert.AreEqual("UO:0000018", uv.ValueArrayUnitCurie); + + var data = reader.GetSpectrumData(212); + Assert.IsNotNull(data); + Assert.AreEqual(96, data!.Mz.Length); + Assert.AreEqual(210.0, data.Mz[0], 1e-4); // value array carries wavelengths + Assert.AreEqual(212.0, data.Mz[1], 1e-4); + Assert.AreEqual(-0.10920, data.Intensity[0], 1e-4); + + // Last wavelength spectrum (global index 731). + Assert.AreEqual("merged=731 row=519", reader.GetSpectrumId(731)); + var last = reader.GetSpectrumData(731); + Assert.IsNotNull(last); + Assert.AreEqual(96, last!.Mz.Length); + Assert.AreEqual(210.0, last.Mz[0], 1e-4); + } + + [TestMethod] + public void Chunked_DecodesToMzPeakNetValues() + { + // mzpeaknet_small_chunked.mzpeak holds the same source data as mzpeaknet_small.mzpeak but in + // the chunked buffer format: one row per m/z chunk, delta-encoded, with seam nulls filled from + // the spectrum's mz_delta_model polynomial. The expected values below were captured from + // mzPeak.NET's own reader of THIS file, so they verify our chunk decoder + model null-fill + + // null-zero intensity bit-exactly (tolerance covers only last-ULP formatting). + using var chunked = new MzPeakReader(SmallChunked); + Assert.AreEqual(48, chunked.SpectrumCount); + + var d0 = chunked.GetSpectrumData(0); + Assert.IsNotNull(d0); + Assert.AreEqual(13589, d0!.Mz.Length); + Assert.AreEqual(202.60657495520474, d0.Mz[0], 1e-9); + Assert.AreEqual(202.60682348271374, d0.Mz[1], 1e-9); + Assert.AreEqual(202.60831465813325, d0.Mz[7], 1e-9, "interpolated (model-filled) m/z"); + Assert.AreEqual(1999.8404377599534, d0.Mz[13588], 1e-9); + Assert.AreEqual(0f, d0.Intensity[0]); + Assert.AreEqual(1938.1174f, d0.Intensity[1], 1e-3f); + Assert.AreEqual(0f, d0.Intensity[7], "null intensity reads back as zero"); + + var d21 = chunked.GetSpectrumData(21); + Assert.IsNotNull(d21); + Assert.AreEqual(11771, d21!.Mz.Length); + // A fill point where the chunked encoding diverges from the row-per-point one (per-chunk vs + // whole-spectrum local-median spacing) — we still match mzPeak.NET's chunked value exactly. + Assert.AreEqual(800.4670743815769, d21.Mz[7221], 1e-9); + } + + [TestMethod] + public void Chunked_HasSamePerSpectrumLengthsAsRowPerPoint() + { + // The two encodings carry the same points (they differ only in floating value at interpolated + // seams), so every spectrum must have the same length and the same total point count. + using var point = new MzPeakReader(Small); + using var chunked = new MzPeakReader(SmallChunked); + + Assert.AreEqual(point.SpectrumCount, chunked.SpectrumCount); + long totalPoints = 0; + for (int i = 0; i < point.SpectrumCount; i++) + { + var p = point.GetSpectrumData(i); + var c = chunked.GetSpectrumData(i); + Assert.AreEqual(p is null, c is null, $"data presence differs at spectrum {i}"); + if (p is null) continue; + Assert.AreEqual(p.Mz.Length, c!.Mz.Length, $"m/z length differs at spectrum {i}"); + Assert.AreEqual(p.Intensity.Length, c.Intensity.Length, $"intensity length differs at spectrum {i}"); + totalPoints += p.Mz.Length; + } + Assert.AreEqual(243054L, totalPoints); + } + + [TestMethod] + public void HasUv_MsSpectraStillReadable_AlongsideWavelengthSpectra() + { + using var reader = new MzPeakReader(HasUv); + + // The MS spectra (indices 0..211) must still read correctly when wavelength spectra are present. + var s0 = reader.GetSpectrumDescription(0); + Assert.AreEqual("scanId=639", s0.Id); + Assert.IsNull(s0.ValueArrayCurie, "MS spectra use m/z (no explicit value-array CURIE)"); + var d0 = reader.GetSpectrumData(0); + Assert.IsNotNull(d0); + Assert.IsTrue(d0!.Mz.Length > 0); + } +} diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/FullMetadataRoundTripTests.cs b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/FullMetadataRoundTripTests.cs new file mode 100644 index 00000000000..745eb6c9108 --- /dev/null +++ b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/FullMetadataRoundTripTests.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using Pwiz.Data.MsData.Diff; +using Pwiz.Data.MsData.Readers; + +namespace Pwiz.Data.MsData.Tests; + +/// +/// Asserts that the mzML-complete binary formats (mzMLb, mzPeak) round-trip the *full* document +/// metadata — not just peak data. Reads the canonical tiny.pwiz.1.1.mzML fixture (rich in +/// instrument configs, component chains, scans, precursors, dataProcessing, param groups, and +/// chromatograms), writes it through each format's writer, reads it back, and requires +/// to be clean (modulo format-inherent annotations the +/// helper tolerates). This runs without any vendor SDK, so it guards metadata fidelity locally — +/// the vendor harness applies the same diff to real vendor documents. +/// +[TestClass] +public class FullMetadataRoundTripTests +{ + private static string FixturePath(string filename) => + Path.Combine(AppContext.BaseDirectory, "example_data", filename); + + private static MSData ReadTiny() + { + var msd = new MSData(); + new MzmlReaderAdapter().Read(FixturePath("tiny.pwiz.1.1.mzML"), msd); + return msd; + } + + [TestMethod] + public void MzMlb_FullMetadata_RoundTrips() + { + var orig = ReadTiny(); + string tmp = Path.Combine(Path.GetTempPath(), $"fullmeta-mzmlb-{Guid.NewGuid():N}.mzMLb"); + try + { + new Pwiz.Data.MsData.MzMlb.MzMlbWriter().Write(orig, tmp); + var rt = new MSData(); + new MzMlbReaderAdapter().Read(tmp, rt); + + string report = MSDataDiff.DescribeRoundTrip(orig, rt, precision: 1.0); + Assert.AreEqual(string.Empty, report, "mzMLb full-metadata round-trip diff:\n" + report); + } + finally { try { File.Delete(tmp); } catch { /* best-effort */ } } + } + + [TestMethod] + public void MzPeak_FullMetadata_RoundTrips() + { + var orig = ReadTiny(); + string tmp = Path.Combine(Path.GetTempPath(), $"fullmeta-mzpeak-{Guid.NewGuid():N}.mzpeak"); + try + { + Pwiz.Data.MsData.MzPeak.WriterMzPeak.Write(orig, tmp); + var rt = new MSData(); + new MzPeakReaderAdapter().Read(tmp, rt); + + string report = MSDataDiff.DescribeRoundTrip(orig, rt, precision: 1.0); + Assert.AreEqual(string.Empty, report, "mzPeak full-metadata round-trip diff:\n" + report); + } + finally { try { File.Delete(tmp); } catch { /* best-effort */ } } + } +} diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/MzPeakReaderAdapterTests.cs b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/MzPeakReaderAdapterTests.cs new file mode 100644 index 00000000000..4ac42758257 --- /dev/null +++ b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/MzPeakReaderAdapterTests.cs @@ -0,0 +1,291 @@ +using System; +using System.IO; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.MsData; +using Pwiz.Data.MsData.MzPeak; +using Pwiz.Data.MsData.Readers; + +namespace Pwiz.Data.MsData.Tests; + +/// +/// IReader-level tests for the mzPeak adapter: write a synthetic file, open it +/// via the adapter, and check that the spectra surface through pwiz's MSData +/// model (SpectrumList / GetSpectrum / Precursors / Scan / BinaryDataArrays). +/// Round-trip behaviour at the column level is covered by +/// ; this layer asserts the column→MSData +/// translation. +/// +[TestClass] +public class MzPeakReaderAdapterTests +{ + private static string s_fixturePath = string.Empty; + + [ClassInitialize] + public static void ClassInit(TestContext _) + { + s_fixturePath = Path.Combine(Path.GetTempPath(), $"mzpeak-adapter-{Guid.NewGuid():N}.mzpeak"); + var spectra = SyntheticData.Spectra(); + var chroms = SyntheticData.Chromatograms(); + MzPeakWriter.Write(s_fixturePath, spectra, SyntheticData.FileMetadata(), chroms); + } + + [ClassCleanup] + public static void ClassCleanup() + { + if (!File.Exists(s_fixturePath)) return; + // The shared fixture is held open by MzPeakArchive for any test that + // leaks its MSData (every `using var msd = ...` above releases it). + // Belt-and-braces: force a finalizer pass + brief retry so teardown + // doesn't fail when a future test forgets the `using`. + for (int attempt = 0; attempt < 5; attempt++) + { + try { File.Delete(s_fixturePath); return; } + catch (IOException) + { + System.GC.Collect(); + System.GC.WaitForPendingFinalizers(); + System.Threading.Thread.Sleep(50); + } + } + // Last attempt — let the exception propagate so a real leak surfaces. + File.Delete(s_fixturePath); + } + + [TestMethod] + public void Identify_RecognisesMzPeakExtension() + { + var adapter = new MzPeakReaderAdapter(); + Assert.AreEqual(CVID.MS_mzPeak_format, adapter.Identify("foo.mzpeak", null)); + Assert.AreEqual(CVID.MS_mzPeak_format, adapter.Identify("foo.MzPeak", null)); + } + + [TestMethod] + public void Identify_RejectsArbitraryFile() + { + var adapter = new MzPeakReaderAdapter(); + Assert.AreEqual(CVID.CVID_Unknown, adapter.Identify("foo.txt", "hello world")); + } + + [TestMethod] + public void Identify_RecognisesMzPeakByMagicWhenExtensionMissing() + { + // ZIP magic prefix forces the secondary archive-content check. + var path = Path.Combine(Path.GetTempPath(), $"mzpeak-magic-{Guid.NewGuid():N}"); + File.Copy(s_fixturePath, path, overwrite: true); + try + { + var head = ReadHead(path, 16); + Assert.AreEqual(CVID.MS_mzPeak_format, new MzPeakReaderAdapter().Identify(path, head)); + } + finally { File.Delete(path); } + } + + [TestMethod] + public void Read_PopulatesSpectrumList() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + + Assert.IsNotNull(msd.Run.SpectrumList); + Assert.AreEqual(3, msd.Run.SpectrumList!.Count); + Assert.AreEqual("synthetic=1 scan=1", msd.Run.SpectrumList.SpectrumIdentity(0).Id); + Assert.AreEqual("synthetic=1 scan=2", msd.Run.SpectrumList.SpectrumIdentity(1).Id); + } + + [TestMethod] + public void Read_PopulatesChromatogramList() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + + Assert.IsNotNull(msd.Run.ChromatogramList); + Assert.AreEqual(3, msd.Run.ChromatogramList!.Count); + Assert.AreEqual("TIC", msd.Run.ChromatogramList.ChromatogramIdentity(0).Id); + } + + [TestMethod] + public void Read_ProfileSpectrum_HasMsLevelAndProfileCv() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + var s = msd.Run.SpectrumList!.GetSpectrum(0, getBinaryData: false); + + Assert.AreEqual(1, s.CvParam(CVID.MS_ms_level).ValueAs()); + Assert.IsTrue(s.HasCVParam(CVID.MS_profile_spectrum)); + Assert.IsFalse(s.HasCVParam(CVID.MS_centroid_spectrum)); + Assert.IsTrue(s.HasCVParam(CVID.MS_positive_scan)); + Assert.AreEqual(400.0, s.CvParam(CVID.MS_base_peak_m_z).ValueAs(), 1e-9); + } + + [TestMethod] + public void Read_Ms2Spectrum_ExposesBothPrecursors() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + var s = msd.Run.SpectrumList!.GetSpectrum(1, getBinaryData: false); + + Assert.AreEqual(2, s.CvParam(CVID.MS_ms_level).ValueAs()); + Assert.AreEqual(2, s.Precursors.Count); + + var p0 = s.Precursors[0]; + Assert.AreEqual(810.79, p0.IsolationWindow.CvParam(CVID.MS_isolation_window_target_m_z).ValueAs(), 1e-9); + Assert.AreEqual(35.0, p0.Activation.CvParam(CVID.MS_collision_energy).ValueAs(), 1e-9); + Assert.IsTrue(p0.Activation.HasCVParam(CVID.MS_collision_induced_dissociation)); + Assert.AreEqual(1, p0.SelectedIons.Count); + Assert.AreEqual(810.7894, p0.SelectedIons[0].CvParam(CVID.MS_selected_ion_m_z).ValueAs(), 1e-9); + Assert.AreEqual(2, p0.SelectedIons[0].CvParam(CVID.MS_charge_state).ValueAs()); + + var p1 = s.Precursors[1]; + Assert.AreEqual(542.21, p1.IsolationWindow.CvParam(CVID.MS_isolation_window_target_m_z).ValueAs(), 1e-9); + Assert.AreEqual(3, p1.SelectedIons[0].CvParam(CVID.MS_charge_state).ValueAs()); + } + + [TestMethod] + public void Read_ScanInfo_ScanStartTimeAndScanWindows() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + var s = msd.Run.SpectrumList!.GetSpectrum(0, getBinaryData: false); + + Assert.AreEqual(1, s.ScanList.Scans.Count); + var scan = s.ScanList.Scans[0]; + Assert.AreEqual(0.005, scan.CvParam(CVID.MS_scan_start_time).ValueAs(), 1e-9); + Assert.AreEqual("FTMS + p ESI Full ms", scan.CvParam(CVID.MS_filter_string).Value); + Assert.AreEqual(2, scan.ScanWindows.Count); + Assert.AreEqual(200.0, scan.ScanWindows[0].CvParam(CVID.MS_scan_window_lower_limit).ValueAs(), 1e-9); + Assert.AreEqual(600.0, scan.ScanWindows[0].CvParam(CVID.MS_scan_window_upper_limit).ValueAs(), 1e-9); + } + + [TestMethod] + public void Read_BinaryData_RoundTripsMZIntensity() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + var s = msd.Run.SpectrumList!.GetSpectrum(0, getBinaryData: true); + + var mz = s.GetMZArray(); + var inten = s.GetIntensityArray(); + Assert.IsNotNull(mz); Assert.IsNotNull(inten); + CollectionAssert.AreEqual(new[] { 100.0, 200.0, 300.0, 400.0, 500.0 }, mz!.Data); + // Intensities were float on the way in; assert with tolerance to absorb the f→d widen. + Assert.AreEqual(5, inten!.Data.Count); + Assert.AreEqual(5e3, inten.Data[3], 1e-6); + Assert.AreEqual(5, s.DefaultArrayLength); + } + + [TestMethod] + public void Read_MetadataOnly_SkipsBinaryData() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + var s = msd.Run.SpectrumList!.GetSpectrum(0, getBinaryData: false); + + // Without binary data the BinaryDataArrays list is empty; the cached + // NumberOfDataPoints from the spectrum metadata still populates + // DefaultArrayLength so downstream code can pre-size buffers. + Assert.AreEqual(0, s.BinaryDataArrays.Count); + Assert.AreEqual(5, s.DefaultArrayLength); + } + + [TestMethod] + public void Read_FillInCommonMetadata_AppendsSourceFile() + { + using var msd = new MSData(); + new MzPeakReaderAdapter().Read(s_fixturePath, msd); + + // FillInCommonMetadata always appends one source-file entry pointing + // at the input. Plus we don't write any source files in our synthetic + // fixture, so this should be exactly one. + Assert.AreEqual(1, msd.FileDescription.SourceFiles.Count); + Assert.AreEqual(Path.GetFileName(s_fixturePath), msd.FileDescription.SourceFiles[0].Name); + + Assert.IsTrue(msd.Software.Any(sw => sw.HasCVParam(CVID.MS_pwiz)), "pwiz software entry should be present"); + } + + private static string ReadHead(string path, int bytes) + { + using var fs = File.OpenRead(path); + byte[] buf = new byte[bytes]; + int n = fs.Read(buf, 0, bytes); + // Latin1 is round-trippable byte→char without transformation for the + // 0x00–0xFF magic bytes we feed into Identify(). + return System.Text.Encoding.Latin1.GetString(buf, 0, n); + } +} + +/// Shared synthetic dataset used by both adapter and round-trip tests. +internal static class SyntheticData +{ + public static MzPeakWriter.SpectrumToWrite[] Spectra() => new[] + { + new MzPeakWriter.SpectrumToWrite( + Index: 0, Id: "synthetic=1 scan=1", Time: 0.0050, MsLevel: 1, IsProfile: true, + Mz: new[] { 100.0, 200.0, 300.0, 400.0, 500.0 }, + Intensity: new[] { 1e3f, 2e3f, 3e3f, 5e3f, 4e3f }, + ScanStartTime: 0.0050, FilterString: "FTMS + p ESI Full ms", + InstrumentConfigurationRef: 0, IonInjectionTime: 68.227, + ScanWindowLowerLimits: new double?[] { 200.0, 800.0 }, + ScanWindowUpperLimits: new double?[] { 600.0, 2000.0 }, + ScanPolarity: 1, + BasePeakMz: 400.0, BasePeakIntensity: 5e3, + TotalIonCurrent: 15e3, + LowestObservedMz: 100.0, HighestObservedMz: 500.0, + SpectrumDataProcessingRef: "DP01"), + new MzPeakWriter.SpectrumToWrite( + Index: 1, Id: "synthetic=1 scan=2", Time: 0.0100, MsLevel: 2, IsProfile: false, + Mz: new[] { 150.5, 250.5, 350.5 }, + Intensity: new[] { 100f, 250f, 80f }, + ScanStartTime: 0.0100, + Precursors: new[] + { + new MzPeakWriter.PrecursorToWrite( + IsolationTargetMz: 810.79, IsolationLowerOffset: 2.0, IsolationUpperOffset: 2.0, + CollisionEnergy: 35.0, DissociationMethodCurie: "MS:1000133", + SelectedIonMz: 810.7894, SelectedIonPeakIntensity: 1234.5, SelectedIonChargeState: 2), + new MzPeakWriter.PrecursorToWrite( + IsolationTargetMz: 542.21, IsolationLowerOffset: 1.5, IsolationUpperOffset: 1.5, + CollisionEnergy: 28.0, DissociationMethodCurie: "MS:1000133", + SelectedIonMz: 542.2105, SelectedIonChargeState: 3), + }), + new MzPeakWriter.SpectrumToWrite( + Index: 2, Id: "synthetic=1 scan=3", Time: 0.0150, MsLevel: 1, IsProfile: true, + Mz: new[] { 110.0, 210.0, 310.0, 410.0 }, + Intensity: new[] { 5e2f, 1.5e3f, 2.5e3f, 1e3f }), + }; + + public static MzPeakWriter.ChromatogramToWrite[] Chromatograms() => new[] + { + new MzPeakWriter.ChromatogramToWrite( + Index: 0, Id: "TIC", ChromatogramTypeCurie: "MS:1000235", DataProcessingRef: "DP01", + Time: new[] { 0.001, 0.002, 0.003 }, + Intensity: new[] { 1e5f, 2e5f, 1.5e5f }), + new MzPeakWriter.ChromatogramToWrite( + Index: 1, Id: "BPC", ChromatogramTypeCurie: "MS:1000628", DataProcessingRef: "DP01", + Time: new[] { 0.001, 0.002, 0.003 }, + Intensity: new[] { 5e4f, 1e5f, 8e4f }), + new MzPeakWriter.ChromatogramToWrite( + Index: 2, Id: "SIM 200.0", ChromatogramTypeCurie: "MS:1000626", DataProcessingRef: "DP01", + Time: new[] { 0.001, 0.002 }, + Intensity: new[] { 100f, 200f }), + }; + + public static MzPeak.FileMetadata FileMetadata() => new( + FileDescription: new MzPeak.FileDescription( + Contents: Array.Empty(), + SourceFiles: Array.Empty()), + InstrumentConfigurations: Array.Empty(), + DataProcessingMethods: Array.Empty(), + Software: Array.Empty(), + Samples: Array.Empty(), + Run: new RunInfo( + Id: "synthetic-run", + DefaultDataProcessingId: "DP01", + DefaultInstrumentId: 0, + DefaultSourceFileId: null, + StartTime: null, + Parameters: Array.Empty()), + SpectrumCount: 3, + SpectrumDataPointCount: 12); +} diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/NonContiguousIndexTests.cs b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/NonContiguousIndexTests.cs new file mode 100644 index 00000000000..d76c0e1d87c --- /dev/null +++ b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/NonContiguousIndexTests.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Pwiz.Data.MsData.MzPeak; + +namespace Pwiz.Data.MsData.Tests; + +/// +/// Regression guard for the data-lookup key: the binary point layer is keyed on the spectrum's +/// stored spectrum.index, NOT its logical row position. The two coincide for pwiz-written +/// files (the writer numbers spectra 0..N-1), so the rest of the suite never exercises the +/// difference. Here we deliberately write non-contiguous indices (7, 42, 100) and confirm each +/// logical spectrum still gets its own peaks back — catching any regression to position-keyed lookup. +/// +[TestClass] +public class NonContiguousIndexTests +{ + [TestMethod] + public void GetSpectrumData_KeysOnStoredIndex_NotRowPosition() + { + var spectra = new[] + { + new MzPeakWriter.SpectrumToWrite( + Index: 7, Id: "scan=7", Time: 0.1, MsLevel: 1, IsProfile: true, + Mz: new[] { 100.0, 101.0 }, Intensity: new[] { 10f, 11f }), + new MzPeakWriter.SpectrumToWrite( + Index: 42, Id: "scan=42", Time: 0.2, MsLevel: 1, IsProfile: true, + Mz: new[] { 200.0, 201.0, 202.0 }, Intensity: new[] { 20f, 21f, 22f }), + new MzPeakWriter.SpectrumToWrite( + Index: 100, Id: "scan=100", Time: 0.3, MsLevel: 1, IsProfile: true, + Mz: new[] { 300.0 }, Intensity: new[] { 30f }), + }; + + var fm = new FileMetadata( + new FileDescription(Array.Empty(), Array.Empty()), + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty(), + new RunInfo("run", null, null, null, null, Array.Empty()), + SpectrumCount: spectra.Length, SpectrumDataPointCount: 6); + + string path = Path.Combine(Path.GetTempPath(), $"mzpeak-noncontig-{Guid.NewGuid():N}.mzpeak"); + try + { + MzPeakWriter.Write(path, spectra, fm); + using var reader = new MzPeakReader(path); + + Assert.AreEqual(3, reader.SpectrumCount); + // Stored indices survive. + Assert.AreEqual(7ul, reader.GetSpectrumDescription(0).Index); + Assert.AreEqual(42ul, reader.GetSpectrumDescription(1).Index); + Assert.AreEqual(100ul, reader.GetSpectrumDescription(2).Index); + + // Each logical spectrum gets ITS peaks (keyed by stored index, not row position). + CollectionAssert.AreEqual(new[] { 100.0, 101.0 }, reader.GetSpectrumData(0)!.Mz); + CollectionAssert.AreEqual(new[] { 200.0, 201.0, 202.0 }, reader.GetSpectrumData(1)!.Mz); + CollectionAssert.AreEqual(new[] { 300.0 }, reader.GetSpectrumData(2)!.Mz); + } + finally { if (File.Exists(path)) File.Delete(path); } + } +} diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/RoundTripTests.cs b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/RoundTripTests.cs new file mode 100644 index 00000000000..7c2493bc249 --- /dev/null +++ b/pwiz-sharp/pwiz/test/MsData.Tests/MzPeak/RoundTripTests.cs @@ -0,0 +1,288 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Pwiz.Data.MsData.MzPeak; + +namespace Pwiz.Data.MsData.Tests; + +/// +/// End-to-end round-trip tests: synthesise a 3-spectrum / 3-chromatogram file +/// (one MS2 with two co-isolated precursors), write it, read it back, and +/// verify every field the writer touched matches what the reader surfaces. +/// One shared fixture file is built in ; each +/// [TestMethod] covers a distinct slice (scan info, multi-precursor, mz_delta_model, +/// chromatograms, ...) so a failure points straight at the broken column group. +/// +[TestClass] +public class RoundTripTests +{ + private static string s_outputPath = string.Empty; + private static MzPeakReader s_reader = null!; + + [ClassInitialize] + public static void ClassInit(TestContext _) + { + s_outputPath = Path.Combine(Path.GetTempPath(), $"mzpeak-roundtrip-{Guid.NewGuid():N}.mzpeak"); + var spectra = BuildSyntheticSpectra(); + var chroms = BuildSyntheticChromatograms(); + MzPeakWriter.Write(s_outputPath, spectra, BuildSyntheticFileMetadata(), chroms); + s_reader = new MzPeakReader(s_outputPath); + } + + [ClassCleanup] + public static void ClassCleanup() + { + s_reader?.Dispose(); + if (File.Exists(s_outputPath)) File.Delete(s_outputPath); + } + + [TestMethod] + public void SpectrumCount_ExcludesPrecursorFanOut() + { + // 3 logical spectra; spectrum 1 fans into 2 parquet rows for its two + // precursors, but the reader collapses that back to a single spectrum. + Assert.AreEqual(3, CountDistinctSpectra(s_reader)); + } + + [TestMethod] + public void Spectrum0_CoreFields_RoundTrip() + { + var d = s_reader.GetSpectrumDescription(0); + Assert.AreEqual("synthetic=1 scan=1", d.Id); + Assert.AreEqual(0.0050, d.Time, 1e-9); + Assert.AreEqual(1, d.MsLevel); + Assert.IsTrue(d.IsProfile); + Assert.IsFalse(d.IsCentroid); + Assert.AreEqual(0, d.Precursors.Count); + } + + [TestMethod] + public void Spectrum0_ScanFields_RoundTrip() + { + var scan = s_reader.GetSpectrumDescription(0).Scan; + Assert.IsNotNull(scan); + Assert.AreEqual(0.0050, scan!.StartTime!.Value, 1e-9); + Assert.AreEqual("FTMS + p ESI Full ms", scan.FilterString); + Assert.AreEqual(0u, scan.InstrumentConfigurationRef); + Assert.AreEqual(68.227, scan.IonInjectionTime!.Value, 1e-9); + Assert.AreEqual(1.234, scan.IonMobilityValue!.Value, 1e-9); + Assert.AreEqual("MS:1002476", scan.IonMobilityTypeCurie); + Assert.AreEqual(2L, scan.PresetScanConfiguration); + } + + [TestMethod] + public void Spectrum0_ScanWindows_RoundTrip() + { + var windows = s_reader.GetSpectrumDescription(0).Scan!.ScanWindows; + Assert.AreEqual(2, windows.Count); + Assert.AreEqual(200.0, windows[0].LowerLimit); + Assert.AreEqual(600.0, windows[0].UpperLimit); + Assert.AreEqual(800.0, windows[1].LowerLimit); + Assert.AreEqual(2000.0, windows[1].UpperLimit); + } + + [TestMethod] + public void Spectrum0_MsDataParityFields_RoundTrip() + { + var d = s_reader.GetSpectrumDescription(0); + Assert.AreEqual(1, d.ScanPolarity); + Assert.AreEqual(400.0, d.BasePeakMz); + Assert.AreEqual(5e3, d.BasePeakIntensity); + Assert.AreEqual(15e3, d.TotalIonCurrent); + Assert.AreEqual(100.0, d.LowestObservedMz); + Assert.AreEqual(500.0, d.HighestObservedMz); + Assert.AreEqual("DP01", d.DataProcessingRef); + Assert.AreEqual(5L, d.NumberOfDataPoints, "auto-populated from Mz.Length"); + } + + [TestMethod] + public void Spectrum0_MzDeltaModel_RoundTrip() + { + var model = s_reader.GetSpectrumDescription(0).MzDeltaModel; + Assert.IsNotNull(model); + CollectionAssert.AreEqual(new[] { 0.001, 0.002, 0.003 }, model!.ToArray()); + } + + [TestMethod] + public void Spectrum1_MultiPrecursor_FansOutToSeparateRows() + { + var d = s_reader.GetSpectrumDescription(1); + Assert.AreEqual(2, d.Precursors.Count); + + var p0 = d.Precursors[0]; + var p1 = d.Precursors[1]; + + // Both precursors share source_index == spectrum.index but carry + // distinct precursor_index values 0 and 1. + Assert.AreEqual(1ul, p0.SourceIndex); + Assert.AreEqual(1ul, p1.SourceIndex); + Assert.AreEqual(0ul, p0.PrecursorIndex); + Assert.AreEqual(1ul, p1.PrecursorIndex); + + Assert.AreEqual("synthetic=1 scan=2", p0.PrecursorId); + Assert.AreEqual(810.79, p0.IsolationWindow!.TargetMz); + Assert.AreEqual(35.0, p0.Activation!.CollisionEnergy); + Assert.AreEqual(810.7894, p0.SelectedIon!.Mz); + + Assert.AreEqual("synthetic=1 scan=2#p2", p1.PrecursorId); + Assert.AreEqual(542.21, p1.IsolationWindow!.TargetMz); + Assert.AreEqual(28.0, p1.Activation!.CollisionEnergy); + Assert.AreEqual(542.2105, p1.SelectedIon!.Mz); + Assert.AreEqual(3L, p1.SelectedIon.ChargeState); + } + + [TestMethod] + public void Spectrum1_SpectrumParameters_PolymorphicValues_RoundTrip() + { + var ps = s_reader.GetSpectrumDescription(1).Parameters; + Assert.AreEqual(4, ps.Count); + Assert.AreEqual("hello", ps.First(p => p.Accession == "MS:9000001").ValueString); + Assert.AreEqual(42L, ps.First(p => p.Accession == "MS:9000002").ValueInteger); + Assert.AreEqual(3.14159, ps.First(p => p.Accession == "MS:9000003").ValueFloat!.Value, 1e-9); + Assert.AreEqual(true, ps.First(p => p.Accession == "MS:9000004").ValueBoolean); + Assert.AreEqual("UO:0000010", ps.First(p => p.Accession == "MS:9000003").Unit); + } + + [TestMethod] + public void SpectrumData_PointArrays_RoundTrip() + { + var d0 = s_reader.GetSpectrumData(0); + Assert.IsNotNull(d0); + CollectionAssert.AreEqual(new[] { 100.0, 200.0, 300.0, 400.0, 500.0 }, d0!.Mz); + + var d1 = s_reader.GetSpectrumData(1); + Assert.IsNotNull(d1); + Assert.AreEqual(3, d1!.Mz.Length); + } + + [TestMethod] + public void SupplementaryPeaks_RoundTrip() + { + var p = s_reader.GetSupplementaryPeaks(0); + Assert.IsNotNull(p); + CollectionAssert.AreEqual(new[] { 200.1, 300.2, 400.3 }, p!.Mz); + + // Spectrum 1 didn't write supplementary peaks. + Assert.IsNull(s_reader.GetSupplementaryPeaks(1)); + } + + [TestMethod] + public void Chromatograms_RoundTrip() + { + Assert.AreEqual(3, s_reader.ChromatogramCount); + var tic = s_reader.GetChromatogramDescription(0); + Assert.AreEqual("TIC", tic.Id); + Assert.AreEqual("MS:1000235", tic.ChromatogramTypeCurie); + + var ticData = s_reader.GetChromatogramData(0); + Assert.IsNotNull(ticData); + Assert.AreEqual(3, ticData!.Time.Length); + } + + // ===== Synthetic fixture builders ===== + + /// + /// Count distinct spectra by their logical index. The reader currently + /// exposes SpectrumCount as the number of primary metadata rows + /// (one per spectrum) — this asserts that the fan-out doesn't inflate it. + /// + private static int CountDistinctSpectra(MzPeakReader r) + { + var seen = new HashSet(); + for (int i = 0; i < r.SpectrumCount; i++) seen.Add(r.GetSpectrumDescription(i).Index); + return seen.Count; + } + + private static MzPeakWriter.SpectrumToWrite[] BuildSyntheticSpectra() => new[] + { + new MzPeakWriter.SpectrumToWrite( + Index: 0, Id: "synthetic=1 scan=1", Time: 0.0050, MsLevel: 1, IsProfile: true, + Mz: new[] { 100.0, 200.0, 300.0, 400.0, 500.0 }, + Intensity: new[] { 1e3f, 2e3f, 3e3f, 5e3f, 4e3f }, + ScanStartTime: 0.0050, FilterString: "FTMS + p ESI Full ms", + InstrumentConfigurationRef: 0, IonInjectionTime: 68.227, + ScanWindowLowerLimits: new double?[] { 200.0, 800.0 }, + ScanWindowUpperLimits: new double?[] { 600.0, 2000.0 }, + SupplementaryPeaksMz: new[] { 200.1, 300.2, 400.3 }, + SupplementaryPeaksIntensity: new[] { 1.9e3f, 2.9e3f, 4.9e3f }, + ScanPolarity: 1, + BasePeakMz: 400.0, BasePeakIntensity: 5e3, + TotalIonCurrent: 15e3, + LowestObservedMz: 100.0, HighestObservedMz: 500.0, + SpectrumDataProcessingRef: "DP01", + ScanIonMobilityValue: 1.234, ScanIonMobilityTypeCurie: "MS:1002476", + PresetScanConfiguration: 2, + MzDeltaModel: new double?[] { 0.001, 0.002, 0.003 }), + new MzPeakWriter.SpectrumToWrite( + Index: 1, Id: "synthetic=1 scan=2", Time: 0.0100, MsLevel: 2, IsProfile: false, + Mz: new[] { 150.5, 250.5, 350.5 }, + Intensity: new[] { 100f, 250f, 80f }, + ScanStartTime: 0.0100, FilterString: "ITMS + c ESI d Full ms2", + InstrumentConfigurationRef: 0, IonInjectionTime: 7.99, + SpectrumParameters: new MzPeakReader.CvParam[] + { + new("custom string", "MS:9000001", ValueString: "hello", null, null, null, Unit: null), + new("custom int", "MS:9000002", null, ValueInteger: 42, null, null, null), + new("custom float", "MS:9000003", null, null, ValueFloat: 3.14159, null, "UO:0000010"), + new("custom bool", "MS:9000004", null, null, null, ValueBoolean: true, null), + }, + ScanParameters: new MzPeakReader.CvParam[] + { + new("scan param 1", "MS:9000010", "alpha", null, null, null, null), + }, + Precursors: new[] + { + new MzPeakWriter.PrecursorToWrite( + PrecursorId: "synthetic=1 scan=2", + IsolationTargetMz: 810.79, IsolationLowerOffset: 2.0, IsolationUpperOffset: 2.0, + CollisionEnergy: 35.0, DissociationMethodCurie: "MS:1000133", + SelectedIonMz: 810.7894, SelectedIonPeakIntensity: 1234.5, SelectedIonChargeState: 2), + new MzPeakWriter.PrecursorToWrite( + PrecursorId: "synthetic=1 scan=2#p2", + IsolationTargetMz: 542.21, IsolationLowerOffset: 1.5, IsolationUpperOffset: 1.5, + CollisionEnergy: 28.0, DissociationMethodCurie: "MS:1000133", + SelectedIonMz: 542.2105, SelectedIonPeakIntensity: 875.0, SelectedIonChargeState: 3), + }), + new MzPeakWriter.SpectrumToWrite( + Index: 2, Id: "synthetic=1 scan=3", Time: 0.0150, MsLevel: 1, IsProfile: true, + Mz: new[] { 110.0, 210.0, 310.0, 410.0 }, + Intensity: new[] { 5e2f, 1.5e3f, 2.5e3f, 1e3f }, + ScanStartTime: 0.0150), + }; + + private static MzPeakWriter.ChromatogramToWrite[] BuildSyntheticChromatograms() => new[] + { + new MzPeakWriter.ChromatogramToWrite( + Index: 0, Id: "TIC", ChromatogramTypeCurie: "MS:1000235", DataProcessingRef: "DP01", + Time: new[] { 0.001, 0.002, 0.003 }, + Intensity: new[] { 1e5f, 2e5f, 1.5e5f }), + new MzPeakWriter.ChromatogramToWrite( + Index: 1, Id: "BPC", ChromatogramTypeCurie: "MS:1000628", DataProcessingRef: "DP01", + Time: new[] { 0.001, 0.002, 0.003 }, + Intensity: new[] { 5e4f, 1e5f, 8e4f }), + new MzPeakWriter.ChromatogramToWrite( + Index: 2, Id: "SIM 200.0", ChromatogramTypeCurie: "MS:1000626", DataProcessingRef: "DP01", + Time: new[] { 0.001, 0.002 }, + Intensity: new[] { 100f, 200f }), + }; + + private static FileMetadata BuildSyntheticFileMetadata() => new( + FileDescription: new FileDescription( + Contents: Array.Empty(), + SourceFiles: Array.Empty()), + InstrumentConfigurations: Array.Empty(), + DataProcessingMethods: Array.Empty(), + Software: Array.Empty(), + Samples: Array.Empty(), + Run: new RunInfo( + Id: "synthetic-run", + DefaultDataProcessingId: "DP01", + DefaultInstrumentId: 0, + DefaultSourceFileId: null, + StartTime: null, + Parameters: Array.Empty()), + SpectrumCount: 3, + SpectrumDataPointCount: 12); +} diff --git a/pwiz-sharp/pwiz/test/MsData.Tests/ReaderListTests.cs b/pwiz-sharp/pwiz/test/MsData.Tests/ReaderListTests.cs index a40be003a4b..e49dcc1484e 100644 --- a/pwiz-sharp/pwiz/test/MsData.Tests/ReaderListTests.cs +++ b/pwiz-sharp/pwiz/test/MsData.Tests/ReaderListTests.cs @@ -14,7 +14,7 @@ public void Default_Registration_AndIdentifyAndIdentifyReader() { var list = ReaderList.Default; - // Default registration order: mzML, mzMLb, mz5, mzXML, MSn, BTDX, MGF. + // Default registration order: mzML, mzMLb, mz5, mzXML, MSn, BTDX, MGF, mzPeak. Assert.AreEqual("mzML", list.Readers[0].TypeName); Assert.AreEqual("mzMLb", list.Readers[1].TypeName); Assert.AreEqual("mz5", list.Readers[2].TypeName); @@ -22,6 +22,7 @@ public void Default_Registration_AndIdentifyAndIdentifyReader() Assert.AreEqual("MSn", list.Readers[4].TypeName); Assert.AreEqual("Bruker Data Exchange", list.Readers[5].TypeName); Assert.AreEqual("MGF", list.Readers[6].TypeName); + Assert.AreEqual("mzPeak", list.Readers[7].TypeName); // Identify: header-sniff wins over filename. const string mzmlHead = "";