pwiz-sharp: added mzPeak format support (read, write, msconvert)

pwiz-sharp/Tools/Commandline/MsConvert/src/ArgParser.cs

@@ -70,6 +70,10 @@ internal static MsConvertConfig Parse(IReadOnlyList<string> args)
                 case "--mzMLb":
                     config.WriteConfig.Format = WriteFormat.MzMLb;
                     break;
+                case "--mzpeak":
+                case "--mzPeak":
+                    config.WriteConfig.Format = WriteFormat.MzPeak;
+                    break;
                 case "--mgf":
                 case "--MGF":
                     config.WriteConfig.Format = WriteFormat.Mgf;
@@ -433,7 +437,9 @@ private static string RequireNext(IReadOnlyList<string> args, ref int i, string
         "  --mzML                   mzML 1.1 (default)",
         "  --mgf                    Mascot Generic Format",
         "  --mzXML                  mzXML 3.2",
-        "  --mz5, --mzMLb, --text (unimplemented)",
+        "  --mzMLb                  mzMLb (HDF5)",
+        "  --mzpeak                 mzPeak (Parquet archive)",
+        "  --mz5, --text (unimplemented)",
         "  --ms1, --bms1, --cms1, --ms2, --bms2, --cms2",
         "",
         "Filters:",

pwiz-sharp/Tools/Commandline/MsConvert/src/Converter.cs

@@ -356,6 +356,7 @@ private string BuildOutputPath(string input, MSData msd)
         WriteFormat.MzXml => ".mzXML",
         WriteFormat.Mz5 => ".mz5",
         WriteFormat.MzMLb => ".mzMLb",
+        WriteFormat.MzPeak => ".mzpeak",
         WriteFormat.Mgf => ".mgf",
         WriteFormat.Text => ".txt",
         WriteFormat.Ms1 => ".ms1",

pwiz-sharp/pwiz/src/MsData/DefaultReaderList.cs

@@ -126,6 +126,11 @@ public static ReaderList Default
             list.Add(new MSnReaderAdapter());
             list.Add(new BtdxReaderAdapter());
             list.Add(new MgfReaderAdapter());
+            // Parquet-backed mzPeak goes at the end of the built-in chain so
+            // it doesn't perturb the existing identify-order for the XML/HDF5
+            // formats. Its extension-first Identify makes the position
+            // immaterial for typed inputs.
+            list.Add(new MzPeakReaderAdapter());
             foreach (var r in AdditionalReaders)
                 list.Add(r);
             return list;

pwiz-sharp/pwiz/src/MsData/Diff/MSDataDiff.cs

@@ -39,6 +39,51 @@ public static string Describe(MSData a, MSData b, DiffConfig? config = null)
         return ctx.Format();
     }
 
+    /// <summary>
+    /// Full-metadata diff for a write→read round-trip through an mzML-complete binary format
+    /// (mzMLb, mzPeak). Compares the whole document at <paramref name="precision"/> tolerance
+    /// (to absorb the format's float32 intensity narrowing) while tolerating the artifacts a
+    /// writer legitimately adds on output that aren't part of the source document:
+    /// <list type="bullet">
+    /// <item>the output file added as a trailing <c>sourceFile</c> self-reference,</item>
+    /// <item>a conversion <c>dataProcessing</c> entry the writer stamps in,</item>
+    /// <item>mzMLb's per-array <c>MS_external_*</c> dataset/offset/length cvParams.</item>
+    /// </list>
+    /// Genuine metadata losses (entries present in <paramref name="original"/> but missing from
+    /// <paramref name="roundtripped"/>) are still reported.
+    /// </summary>
+    public static string DescribeRoundTrip(MSData original, MSData roundtripped, double precision)
+    {
+        ArgumentNullException.ThrowIfNull(original);
+        ArgumentNullException.ThrowIfNull(roundtripped);
+
+        // Drop writer-added sourceFiles / dataProcessings (present in the round-tripped doc but
+        // not the source — keyed by decoded id). Real losses surface as a-only and are untouched.
+        var origSourceFileIds = original.FileDescription.SourceFiles
+            .Select(s => DecodeXmlId(s.Id)).ToHashSet(StringComparer.Ordinal);
+        roundtripped.FileDescription.SourceFiles
+            .RemoveAll(s => !origSourceFileIds.Contains(DecodeXmlId(s.Id)));
+
+        // De-duplicate dataProcessings by id first: MSDataFile.FillInCommonMetadata stamps a
+        // conversion entry on every read, so a round-tripped doc carries it twice (once embedded
+        // from the source, once re-added on read-back) under the same id. Then drop any remaining
+        // writer-added entries not present in the source.
+        DedupeById(roundtripped.DataProcessings, d => d.Id);
+        var origDataProcessingIds = original.DataProcessings
+            .Select(d => DecodeXmlId(d.Id)).ToHashSet(StringComparer.Ordinal);
+        roundtripped.DataProcessings
+            .RemoveAll(d => !origDataProcessingIds.Contains(DecodeXmlId(d.Id)));
+
+        var dc = new DiffConfig { Precision = precision, IgnoreVersions = true };
+        return Describe(original, roundtripped, dc);
+    }
+
+    private static void DedupeById<T>(List<T> items, Func<T, string> keyOf)
+    {
+        var seen = new HashSet<string>(StringComparer.Ordinal);
+        items.RemoveAll(item => !seen.Add(DecodeXmlId(keyOf(item))));
+    }
+
     /// <summary>
     /// Tolerance mode for the <c>msLevel</c> comparison in <see cref="DescribeSpectraDataOnly"/>.
     /// Captures the well-known lossy defaults each peak-list format applies on read.
@@ -412,8 +457,8 @@ private static void DiffMobilityTriples(Spectrum a, Spectrum b, Context ctx)
 
     private static void DiffArrayMetadata(BinaryDataArray a, BinaryDataArray b, Context ctx)
     {
-        var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList();
-        var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList();
+        var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList();
+        var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList();
         DiffCvParamLists(aCvParams, bCvParams, ctx);
         DiffUserParamLists(a.UserParams, b.UserParams, ctx);
     }
@@ -520,8 +565,8 @@ private static void DiffBinaryArray(BinaryDataArray a, BinaryDataArray b, Contex
     {
         // Filter out purely-serialization cvParams (32-bit / 64-bit precision, compression type)
         // before diffing — they describe how the array was encoded, not its content.
-        var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList();
-        var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList();
+        var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList();
+        var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList();
         DiffCvParamLists(aCvParams, bCvParams, ctx);
         // UserParams and ParamGroup refs still matter; compare them raw.
         DiffUserParamLists(a.UserParams, b.UserParams, ctx);
@@ -541,6 +586,16 @@ private static void DiffBinaryArray(BinaryDataArray a, BinaryDataArray b, Contex
         }
     }
 
+    // mzMLb stores binary arrays out-of-line in HDF5 and points to them with these per-array
+    // cvParams (dataset name + offset + length). They're an internal mzMLb mechanism, not source
+    // metadata, so a round-trip diff against a non-mzMLb source must ignore them.
+    private static bool IsExternalBinaryRefCv(CVID cvid)
+    {
+        return cvid is CVID.MS_external_HDF5_dataset
+                    or CVID.MS_external_offset
+                    or CVID.MS_external_array_length;
+    }
+
     private static bool IsBinaryEncodingCv(CVID cvid)
     {
         return cvid is CVID.MS_32_bit_float
@@ -578,8 +633,8 @@ private static void DiffUserParamLists(List<UserParam> a, List<UserParam> b, Con
     private static void DiffIntegerArray(IntegerDataArray a, IntegerDataArray b, Context ctx)
     {
         // Same encoding-CV filtering as BinaryDataArray — int32/int64 choice is serialization.
-        var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList();
-        var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid)).ToList();
+        var aCvParams = a.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList();
+        var bCvParams = b.CVParams.Where(p => !IsBinaryEncodingCv(p.Cvid) && !IsExternalBinaryRefCv(p.Cvid)).ToList();
         DiffCvParamLists(aCvParams, bCvParams, ctx);
         DiffUserParamLists(a.UserParams, b.UserParams, ctx);
 

pwiz-sharp/pwiz/src/MsData/MSDataFile.cs

@@ -168,6 +168,13 @@ public static void Write(MSData msd, string path, WriteConfig config, IterationL
             return;
         }
 
+        // mzPeak is also path-bound: a ZIP archive of Parquet tables built via random file I/O.
+        if (config.Format == WriteFormat.MzPeak)
+        {
+            Pwiz.Data.MsData.MzPeak.WriterMzPeak.Write(msd, path);
+            return;
+        }
+
         using Stream output = OpenOutputStream(path, config.Gzip);
         Write(msd, output, config, ilr);
     }

pwiz-sharp/pwiz/src/MsData/MsData.csproj

@@ -26,6 +26,10 @@
          to read/write HDF5-backed mass-spec files. The 1.10 stream tracks
          HDF5 1.10.x — the same major version pwiz cpp builds against. -->
     <PackageReference Include="HDF.PInvoke.1.10" Version="1.10.612" />
+    <!-- ParquetSharp wraps Apache Arrow's parquet-cpp with bundled native
+         binaries for win-x64 / linux-x64. Used by Reader_MzPeak / WriterMzPeak
+         for the nested-struct GroupNode schema mzPeak emits. -->
+    <PackageReference Include="ParquetSharp" Version="23.0.0.2" />
   </ItemGroup>
 
 </Project>

pwiz-sharp/pwiz/src/MsData/MzPeak/ChromatogramList_MzPeak.cs

@@ -0,0 +1,191 @@
+using System;
+using Pwiz.Data.Common.Cv;
+using Pwiz.Data.MsData.Processing;
+using Pwiz.Data.MsData.Spectra;
+
+namespace Pwiz.Data.MsData.MzPeak;
+
+/// <summary>
+/// Lazy <see cref="IChromatogramList"/> over an <see cref="MzPeakReader"/>.
+/// Sibling of <see cref="SpectrumList_MzPeak"/>. The mzPeak chromatogram model
+/// is much simpler than the spectrum model — id, type CURIE, and the
+/// (time, intensity) point arrays — so the translation here is short.
+/// </summary>
+internal sealed class ChromatogramList_MzPeak : ChromatogramListBase
+{
+    private readonly MzPeakReader _reader;
+    private readonly bool _ownsReader;
+    private readonly DataProcessing? _dp;
+    private readonly ChromatogramIdentity[] _identities;
+
+    public ChromatogramList_MzPeak(MzPeakReader reader, DataProcessing? dp, bool ownsReader)
+    {
+        ArgumentNullException.ThrowIfNull(reader);
+        _reader = reader;
+        _ownsReader = ownsReader;
+        _dp = dp;
+        _identities = new ChromatogramIdentity[reader.ChromatogramCount];
+        for (int i = 0; i < reader.ChromatogramCount; i++)
+        {
+            var desc = reader.GetChromatogramDescription(i);
+            _identities[i] = new ChromatogramIdentity { Index = i, Id = desc.Id };
+        }
+    }
+
+    public override int Count => _identities.Length;
+
+    public override ChromatogramIdentity ChromatogramIdentity(int index) => _identities[index];
+
+    public override DataProcessing? DataProcessing => _dp;
+
+    public override Chromatogram GetChromatogram(int index, bool getBinaryData = false)
+    {
+        if ((uint)index >= (uint)_identities.Length)
+            throw new ArgumentOutOfRangeException(nameof(index));
+
+        var desc = _reader.GetChromatogramDescription(index);
+        var chrom = new Chromatogram
+        {
+            Index = index,
+            Id = desc.Id,
+        };
+
+        // Chromatogram type CURIE → CV term. mzPeak stores e.g. MS:1000235
+        // (total ion current chromatogram). Set both the bag-level type
+        // tag and any chromatogram-type-specific defaults the cpp readers emit.
+        var typeCvid = CvidFromCurie(desc.ChromatogramTypeCurie);
+        if (typeCvid != CVID.CVID_Unknown)
+            chrom.Params.Set(typeCvid);
+
+        if (desc.Parameters is not null)
+            ApplyParams(chrom.Params, desc.Parameters);
+
+        if (getBinaryData)
+        {
+            // pwiz/mzML chromatograms always carry a time + intensity array pair, even when empty;
+            // emit them unconditionally so 0-point chromatograms round-trip with the right shape.
+            var data = _reader.GetChromatogramData(index);
+            var time = data?.Time ?? Array.Empty<double>();
+            var intensitySrc = data?.Intensity ?? Array.Empty<float>();
+            var intensityDouble = new double[intensitySrc.Length];
+            for (int i = 0; i < intensitySrc.Length; i++) intensityDouble[i] = intensitySrc[i];
+            var timeUnit = CvidFromCurie(desc.TimeUnitCurie);
+            if (timeUnit == CVID.CVID_Unknown) timeUnit = CVID.UO_minute;
+            var intensityUnit = CvidFromCurie(desc.IntensityUnitCurie);
+            if (intensityUnit == CVID.CVID_Unknown) intensityUnit = CVID.MS_number_of_detector_counts;
+            SetTimeIntensityArrays(chrom, time, intensityDouble, timeUnit, intensityUnit);
+            MzPeakAuxArrays.Apply(desc.AuxArrays, chrom.BinaryDataArrays, chrom.IntegerDataArrays);
+        }
+
+        return chrom;
+    }
+
+    protected override void DisposeCore()
+    {
+        if (_ownsReader) _reader.Dispose();
+    }
+
+    /// <summary>
+    /// pwiz's <see cref="Chromatogram"/> doesn't expose a SetTimeIntensityArrays
+    /// helper (asymmetric with <see cref="Spectrum.SetMZIntensityArrays"/>), so
+    /// build the two BinaryDataArrays explicitly. Time uses MS_minute (matching
+    /// the cpp <c>Chromatogram::set_time_intensity_arrays</c> default).
+    /// </summary>
+    private static void SetTimeIntensityArrays(Chromatogram chrom, double[] time, double[] intensity, CVID timeUnit, CVID intensityUnit)
+    {
+        var timeArr = new BinaryDataArray();
+        timeArr.Set(CVID.MS_time_array, "", timeUnit);
+        timeArr.Data.AddRange(time);
+        chrom.BinaryDataArrays.Add(timeArr);
+
+        var intArr = new BinaryDataArray();
+        intArr.Set(CVID.MS_intensity_array, "", intensityUnit);
+        intArr.Data.AddRange(intensity);
+        chrom.BinaryDataArrays.Add(intArr);
+
+        chrom.DefaultArrayLength = time.Length;
+    }
+
+    private static CVID CvidFromCurie(string? curie)
+    {
+        if (string.IsNullOrEmpty(curie)) return CVID.CVID_Unknown;
+        return CvLookup.CvTermInfo(curie).Cvid;
+    }
+
+    /// <summary>Apply free-form chromatogram params (CV → CVParam, else UserParam, keeping type).</summary>
+    private static void ApplyParams(Pwiz.Data.Common.Params.ParamContainer target, System.Collections.Generic.IReadOnlyList<MzPeakReader.CvParam> src)
+    {
+        foreach (var p in src)
+        {
+            var cvid = CvidFromCurie(p.Accession);
+            var unitCvid = string.IsNullOrEmpty(p.Unit) ? CVID.CVID_Unknown : CvidFromCurie(p.Unit);
+            string value = p.ValueString
+                ?? p.ValueInteger?.ToString(System.Globalization.CultureInfo.InvariantCulture)
+                ?? p.ValueFloat?.ToString("R", System.Globalization.CultureInfo.InvariantCulture)
+                ?? (p.ValueBoolean is bool b ? (b ? "true" : "false") : string.Empty);
+
+            if (cvid != CVID.CVID_Unknown)
+                target.Set(cvid, value, unitCvid);
+            else
+                target.UserParams.Add(new Pwiz.Data.Common.Params.UserParam(
+                    p.Name ?? string.Empty, value, type: p.Type ?? string.Empty, units: unitCvid));
+        }
+    }
+}
+
+/// <summary>
+/// Rebuilds auxiliary (non-canonical) binary/integer data arrays from their round-tripped
+/// <see cref="AuxiliaryArrayData"/> records onto a spectrum or chromatogram. Shared by
+/// <see cref="SpectrumList_MzPeak"/> and <see cref="ChromatogramList_MzPeak"/>.
+/// </summary>
+internal static class MzPeakAuxArrays
+{
+    public static void Apply(
+        System.Collections.Generic.IReadOnlyList<AuxiliaryArrayData>? aux,
+        System.Collections.Generic.List<BinaryDataArray> binaryTarget,
+        System.Collections.Generic.List<IntegerDataArray> integerTarget)
+    {
+        if (aux is null) return;
+        foreach (var a in aux)
+        {
+            if (a.IsInteger)
+            {
+                var arr = new IntegerDataArray();
+                ApplyMzPeakParams(arr, a.Params);
+                if (a.IntValues is not null) arr.Data.AddRange(a.IntValues);
+                integerTarget.Add(arr);
+            }
+            else
+            {
+                var arr = new BinaryDataArray();
+                ApplyMzPeakParams(arr, a.Params);
+                if (a.DoubleValues is not null) arr.Data.AddRange(a.DoubleValues);
+                binaryTarget.Add(arr);
+            }
+        }
+    }
+
+    internal static void ApplyMzPeakParams(Pwiz.Data.Common.Params.ParamContainer target,
+        System.Collections.Generic.IReadOnlyList<MzPeakCvParam> src)
+    {
+        foreach (var p in src)
+        {
+            var cvid = string.IsNullOrEmpty(p.Accession) ? CVID.CVID_Unknown : CvLookup.CvTermInfo(p.Accession!).Cvid;
+            var unit = string.IsNullOrEmpty(p.Unit) ? CVID.CVID_Unknown : CvLookup.CvTermInfo(p.Unit!).Cvid;
+            string value = p.Value switch
+            {
+                null => string.Empty,
+                string s => s,
+                bool b => b ? "true" : "false",
+                long l => l.ToString(System.Globalization.CultureInfo.InvariantCulture),
+                int i => i.ToString(System.Globalization.CultureInfo.InvariantCulture),
+                double d => d.ToString("R", System.Globalization.CultureInfo.InvariantCulture),
+                _ => p.Value.ToString() ?? string.Empty,
+            };
+            if (cvid != CVID.CVID_Unknown)
+                target.Set(cvid, value, unit);
+            else
+                target.UserParams.Add(new Pwiz.Data.Common.Params.UserParam(p.Name ?? string.Empty, value, type: string.Empty, units: unit));
+        }
+    }
+}