diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2211f89 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Vendored Osprey.ML sources are byte-for-byte copies of their upstream in pwiz, CRLF +# line endings included, and MARS.Test hashes them against UPSTREAM.json to turn an +# accidental local edit into a test failure. Git's autocrlf translation would make that +# hash platform-dependent - passing on Windows, failing on Linux CI - so these files must +# never be translated. Treat them as opaque bytes. +dotnet/third_party/** -text diff --git a/.github/workflows/dotnet-release.yml b/.github/workflows/dotnet-release.yml new file mode 100644 index 0000000..d68a6b9 --- /dev/null +++ b/.github/workflows/dotnet-release.yml @@ -0,0 +1,246 @@ +name: .NET Release + +# Pushing a v* tag builds every platform artifact and creates the GitHub Release. +# Do not hand-create the Release: the notes file is published verbatim as the body, and +# this workflow is what guarantees the tag, the assembly version and the notes agree. +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Version to build, without the v prefix (e.g. 26.1.0). Builds artifacts only, creates no Release.' + required: true + +permissions: + contents: write + +jobs: + # Fail before spending twenty minutes on artifacts if the release is not consistent. + # Catching this after the builds have run is the failure mode the process doc warns about. + preflight: + name: Check version and notes + runs-on: ubuntu-latest + outputs: + version: ${{ steps.resolve.outputs.version }} + tag: ${{ steps.resolve.outputs.tag }} + steps: + - uses: actions/checkout@v4 + + - name: Resolve version + id: resolve + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + version="${{ github.event.inputs.version }}" + else + version="${GITHUB_REF_NAME#v}" + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "tag=v$version" >> "$GITHUB_OUTPUT" + echo "Releasing version $version" + + - name: Assembly version must match the tag + run: | + version="${{ steps.resolve.outputs.version }}" + declared=$(grep -oPm1 '(?<=)[^<]+' dotnet/Directory.Build.props) + echo "tag says $version" + echo "csproj says $declared" + if [ "$declared" != "$version" ]; then + echo "::error::dotnet/Directory.Build.props declares $declared but the tag says $version. Bump it and re-tag." + exit 1 + fi + + - name: Release notes must exist + if: github.event_name != 'workflow_dispatch' + run: | + notes="release-notes/RELEASE_NOTES_${{ steps.resolve.outputs.tag }}.md" + if [ ! -f "$notes" ]; then + echo "::error::$notes is missing. Rename RELEASE_NOTES_next.md to it before tagging; it is published verbatim as the Release description." + exit 1 + fi + # A renamed draft always arrives carrying the headings nobody filled in, and this + # file is published verbatim as the Release description. + empty=$(awk ' + /^## / { if (heading != "" && count == 0) print heading; heading=$0; count=0; next } + NF { count++ } + END { if (heading != "" && count == 0) print heading }' "$notes") + if [ -n "$empty" ]; then + echo "::error::$notes has section headings with nothing under them:" + echo "$empty" + echo "Delete them before tagging - this file is the Release description." + exit 1 + fi + echo "Release notes look complete." + + build: + name: ${{ matrix.rid }} + needs: preflight + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Each artifact is built on its own platform where a runner exists, so the + # smoke test below exercises the binary that actually ships rather than a + # cross-compiled stand-in. + - { os: windows-latest, rid: win-x64, archive: zip } + - { os: ubuntu-latest, rid: linux-x64, archive: tar } + - { os: macos-latest, rid: osx-arm64, archive: tar } + - { os: macos-13, rid: osx-x64, archive: tar } + # No hosted arm64 runner in the standard pool for either of these, so they are + # cross-compiled and NOT smoke tested. GitHub's windows-11-arm runner can be + # swapped in for win-arm64 if it becomes available to this repository. + - { os: windows-latest, rid: win-arm64, archive: zip, cross: true } + - { os: ubuntu-latest, rid: linux-arm64, archive: tar, cross: true } + + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + # Released binaries carry the vendor readers, so a downloaded MARS opens a .raw with + # nothing else installed. That needs a pwiz-sharp checkout at the pinned commit, because + # it is an unmerged draft with no package feed. + - name: Read the pinned pwiz-sharp commit + id: pin + shell: bash + run: | + commit=$(python -c "import json;print(json.load(open('dotnet/pwiz-sharp.json'))['commit'])") + echo "commit=$commit" >> "$GITHUB_OUTPUT" + + - name: Cache the pwiz checkout + id: pwiz-cache + uses: actions/cache@v4 + with: + path: pwiz + key: pwiz-sharp-${{ runner.os }}-${{ steps.pin.outputs.commit }} + + - name: Clone pwiz-sharp at the pin + if: steps.pwiz-cache.outputs.cache-hit != 'true' + shell: bash + run: | + # Blobless but a full working tree: Bruker reads archives from pwiz_aux and VC90 CRT + # files from pwiz_tools/Shared/Lib, and Common.csproj embeds .obo files from + # pwiz/data/common. + git clone --filter=blob:none --no-checkout https://github.com/ProteoWizard/pwiz.git pwiz + cd pwiz + git checkout --detach ${{ steps.pin.outputs.commit }} + + - name: Pin the SDK for pwiz-sharp + shell: bash + run: | + cat > pwiz/pwiz-sharp/global.json <<'JSON' + { "sdk": { "version": "8.0.0", "rollForward": "latestFeature" } } + JSON + + - name: Publish + working-directory: dotnet + shell: bash + run: | + dotnet publish MARS/MARS.csproj -c Release -f net8.0 -r ${{ matrix.rid }} --self-contained true -p:PublishSingleFile=true -p:DebugType=none -p:PwizSharpDir="$GITHUB_WORKSPACE/pwiz/pwiz-sharp" -p:IAgreeToVendorLicenses=true -o publish/${{ matrix.rid }} + + # Parquet.Net carries a native compression library, so the archive is a binary plus + # that library rather than a lone executable. + - name: List payload + working-directory: dotnet/publish/${{ matrix.rid }} + shell: bash + run: ls -la + + - name: Smoke test + if: ${{ !matrix.cross }} + working-directory: dotnet/publish/${{ matrix.rid }} + shell: bash + run: | + if [ "${{ runner.os }}" = "Windows" ]; then bin=./mars.exe; else bin=./mars; chmod +x "$bin"; fi + "$bin" --version + # --version alone would pass on a binary whose dependencies are broken, because + # it never touches them. Run a command that opens a file and does real work. + "$bin" verify --help > /dev/null + "$bin" qc --help > /dev/null + + # A build that silently missed pwiz-sharp compiles, runs and writes mzML, so it + # would pass everything above having shipped none of the vendor support. --version + # reports what this binary actually carries. + capabilities=$("$bin" --version) + + # Only what every runtime identifier can do. Bruker and Sciex ship native x64 + # libraries and mzMLb needs a native HDF5, so an arm64 artifact honestly reports + # fewer of them - osx-arm64 runs this check, and demanding .d here would fail a + # release that was working exactly as intended. Thermo's SDK is managed and mzXML is + # pure pwiz, so those two prove pwiz-sharp linked in on any target. + for expected in .raw mzXML; do + if ! echo "$capabilities" | grep -qF -- "$expected"; then + echo "::error::$expected missing from ${{ matrix.rid }} - vendor support did not build in" + exit 1 + fi + done + echo "Smoke test passed on ${{ matrix.rid }}, with vendor support" + + - name: Archive (tar.gz) + if: matrix.archive == 'tar' + working-directory: dotnet/publish/${{ matrix.rid }} + run: | + chmod +x mars + tar czf "../../../mars-${{ needs.preflight.outputs.version }}-${{ matrix.rid }}.tar.gz" . + + - name: Archive (zip) + if: matrix.archive == 'zip' + working-directory: dotnet/publish/${{ matrix.rid }} + shell: pwsh + run: | + Compress-Archive -Path * -DestinationPath "$env:GITHUB_WORKSPACE/mars-${{ needs.preflight.outputs.version }}-${{ matrix.rid }}.zip" + + - name: Checksum + shell: bash + run: | + cd "$GITHUB_WORKSPACE" + f=$(ls mars-${{ needs.preflight.outputs.version }}-${{ matrix.rid }}.*) + if command -v sha256sum > /dev/null; then sha256sum "$f" > "$f.sha256"; else shasum -a 256 "$f" > "$f.sha256"; fi + cat "$f.sha256" + + - uses: actions/upload-artifact@v4 + with: + name: mars-${{ matrix.rid }} + path: | + mars-*.tar.gz + mars-*.zip + mars-*.sha256 + if-no-files-found: error + + release: + name: Publish GitHub Release + needs: [preflight, build] + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Collect checksums + run: | + cd artifacts + ls -la + cat *.sha256 > SHA256SUMS.txt + rm -f *.sha256 + cat SHA256SUMS.txt + + # The notes file is the Release body, verbatim. Preflight has already confirmed it + # exists and carries no empty headings. + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + name: MARS ${{ needs.preflight.outputs.version }} + body_path: release-notes/RELEASE_NOTES_${{ needs.preflight.outputs.tag }}.md + files: | + artifacts/*.tar.gz + artifacts/*.zip + artifacts/SHA256SUMS.txt + fail_on_unmatched_files: true diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml new file mode 100644 index 0000000..32cdb13 --- /dev/null +++ b/.github/workflows/dotnet.yml @@ -0,0 +1,281 @@ +name: .NET + +on: + push: + branches: [main] + paths: + - 'dotnet/**' + - '.github/workflows/dotnet.yml' + pull_request: + branches: [main] + paths: + - 'dotnet/**' + - '.github/workflows/dotnet.yml' + workflow_dispatch: + +jobs: + build: + name: Build and test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore + working-directory: dotnet + run: dotnet restore + + - name: Build + working-directory: dotnet + run: dotnet build --no-restore -c Release -warnaserror + + - name: Test + working-directory: dotnet + run: dotnet test --no-build -c Release --logger "console;verbosity=normal" + + # Vendor formats: Thermo, Bruker and Sciex reading, and mzXML / mzMLb / mgf writing. + # + # Separate from the matrix above because it needs a pwiz-sharp checkout, which is a draft + # branch with no package feed. Everything else in this workflow builds the configuration + # MARS ships today - mzML only - and that configuration has to keep working, so it stays the + # default rather than becoming a special case. + # + # Windows because it is the only platform where all three vendors work: the Sciex SDK is + # Windows-only. Thermo and Bruker also build on Linux; adding that is worth doing once this + # one has settled. + pwiz: + name: Vendor formats (pwiz-sharp) + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Read the pinned pwiz-sharp commit + id: pin + shell: bash + run: | + commit=$(python -c "import json;print(json.load(open('dotnet/pwiz-sharp.json'))['commit'])") + echo "commit=$commit" >> "$GITHUB_OUTPUT" + echo "Pinned to $commit" + + # Keyed on the commit, so a pinned build is fetched once and reused until the pin moves. + # The tree is several gigabytes: without this every run would pay for it again. + - name: Cache the pwiz checkout + id: pwiz-cache + uses: actions/cache@v4 + with: + path: pwiz + key: pwiz-sharp-${{ steps.pin.outputs.commit }} + + - name: Clone pwiz-sharp at the pin + if: steps.pwiz-cache.outputs.cache-hit != 'true' + shell: bash + run: | + # Blobless: the history is not needed, only the tree at one commit. The FULL working + # tree is needed though - Bruker reads archives from pwiz_aux and VC90 CRT files from + # pwiz_tools/Shared/Lib, and Common.csproj embeds .obo files from pwiz/data/common. + git clone --filter=blob:none --no-checkout https://github.com/ProteoWizard/pwiz.git pwiz + cd pwiz + git checkout --detach ${{ steps.pin.outputs.commit }} + + # pwiz-sharp does not carry a global.json, so a runner with a newer SDK fails to resolve + # one for the nested `dotnet run` that generates the vendor pins. Written here rather + # than committed upstream because it is our build's problem to solve. + - name: Pin the SDK for pwiz-sharp + shell: bash + run: | + cat > pwiz/pwiz-sharp/global.json <<'JSON' + { "sdk": { "version": "8.0.0", "rollForward": "latestFeature" } } + JSON + + - name: Build with vendor support + working-directory: dotnet + shell: bash + run: | + dotnet build -c Release -warnaserror -p:PwizSharpDir="$GITHUB_WORKSPACE/pwiz/pwiz-sharp" -p:IAgreeToVendorLicenses=true + + - name: Test with vendor support + working-directory: dotnet + shell: bash + run: | + dotnet test --no-build -c Release --logger "console;verbosity=normal" -p:PwizSharpDir="$GITHUB_WORKSPACE/pwiz/pwiz-sharp" -p:IAgreeToVendorLicenses=true + + # The formats are the point of the job, so prove the binary can actually produce them + # rather than only that the code compiles. + - name: Publish and check the formats are offered + working-directory: dotnet + shell: bash + run: | + dotnet publish MARS/MARS.csproj -c Release -f net8.0 -r win-x64 --self-contained true -p:PublishSingleFile=true -p:PwizSharpDir="$GITHUB_WORKSPACE/pwiz/pwiz-sharp" -p:IAgreeToVendorLicenses=true -o "$GITHUB_WORKSPACE/vendor-build" + + cd "$GITHUB_WORKSPACE/vendor-build" + capabilities=$(./mars.exe --version) + echo "$capabilities" + + # --version reports what THIS binary carries. A build that silently missed + # pwiz-sharp still compiles, still runs, and still writes mzML, so without a check + # here the job would go green having proved nothing. + for expected in .raw .d .wiff2 mzXML mzMLb mgf; do + if ! echo "$capabilities" | grep -qF -- "$expected"; then + echo "::error::$expected missing - this build did not pick up pwiz-sharp" + exit 1 + fi + done + echo "Vendor reading and every output format are present." + + # Produces the same artifacts a release would, on every push and pull request, so a + # build is downloadable without cutting a release and so packaging breaks surface here + # rather than at tag time. + package: + name: Package ${{ matrix.rid }} + needs: build + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - { os: windows-latest, rid: win-x64, archive: zip } + - { os: ubuntu-latest, rid: linux-x64, archive: tar } + - { os: macos-latest, rid: osx-arm64, archive: tar } + - { os: macos-13, rid: osx-x64, archive: tar } + # No hosted arm64 runner in the standard pool for either of these, so they are + # cross-compiled and NOT smoke tested. GitHub's windows-11-arm runner can be + # swapped in for win-arm64 if it becomes available to this repository. + - { os: windows-latest, rid: win-arm64, archive: zip, cross: true } + - { os: ubuntu-latest, rid: linux-arm64, archive: tar, cross: true } + + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Publish self-contained + working-directory: dotnet + run: > + dotnet publish MARS/MARS.csproj + -c Release -f net8.0 -r ${{ matrix.rid }} + --self-contained true + -p:PublishSingleFile=true + -p:DebugType=none + -o publish/${{ matrix.rid }} + + - name: Smoke test + if: ${{ !matrix.cross }} + working-directory: dotnet/publish/${{ matrix.rid }} + shell: bash + run: | + if [ "${{ runner.os }}" = "Windows" ]; then bin=./mars.exe; else bin=./mars; chmod +x "$bin"; fi + "$bin" --version + "$bin" verify --help > /dev/null + "$bin" qc --help > /dev/null + + - name: Archive (tar.gz) + if: matrix.archive == 'tar' + working-directory: dotnet/publish/${{ matrix.rid }} + run: | + chmod +x mars + tar czf "../../../mars-${{ matrix.rid }}.tar.gz" . + + - name: Archive (zip) + if: matrix.archive == 'zip' + working-directory: dotnet/publish/${{ matrix.rid }} + shell: pwsh + run: Compress-Archive -Path * -DestinationPath "$env:GITHUB_WORKSPACE/mars-${{ matrix.rid }}.zip" + + - uses: actions/upload-artifact@v4 + with: + name: mars-${{ matrix.rid }} + path: | + mars-*.tar.gz + mars-*.zip + if-no-files-found: error + retention-days: 14 + + # MARS targets net8.0 by default, which runs unchanged on the .NET 10 runtime. Building + # AGAINST net10.0 is the forward-looking case the port spec asks for. It is not the + # target that ships, so it reports rather than gates. + net10: + name: Build and test against net10.0 (informational) + runs-on: ubuntu-latest + continue-on-error: true + + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 8.0.x + 10.0.x + + # A semicolon-separated list cannot be passed on the command line: the shell strips + # the quotes and MSBuild reads net10.0 as a second property, and escaping it as %3B + # instead makes MSBuild treat the whole string as one target framework. + # MarsIncludeNet10 selects the pair inside Directory.Build.props. + - name: Build + working-directory: dotnet + run: dotnet build -c Release -p:MarsIncludeNet10=true + + - name: Test + working-directory: dotnet + run: > + dotnet test --no-build -c Release -f net10.0 + -p:MarsIncludeNet10=true + --logger "console;verbosity=normal" + + determinism: + name: Determinism + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Build + working-directory: dotnet + run: dotnet build -c Release + + # MARS writes m/z values into files that get reprocessed and compared, so identical + # input must give a bit-identical output at any thread count. Broken out as its own + # job so a failure here is unmistakable rather than one line in a long test log. + - name: Same input, same bytes, any thread count + working-directory: dotnet + run: dotnet test --no-build -c Release --filter "FullyQualifiedName~Deterministic" + + vendor-drift: + name: Vendored Osprey.ML drift guard + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + # Osprey.ML owns the boosting code; MARS carries a hash-guarded copy. This fails + # when someone edits the copy instead of fixing it upstream in pwiz. + - name: Check vendored hashes + working-directory: dotnet + run: dotnet test -c Release --filter "FullyQualifiedName~VendoredOsprey" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index deedc8d..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Publish to PyPI - -on: - release: - types: [published] - -jobs: - build: - name: Build distribution - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install build dependencies - run: python -m pip install --upgrade pip build - - - name: Build package - run: python -m build - - - name: Store distribution packages - uses: actions/upload-artifact@v6 - with: - name: python-package-distributions - path: dist/ - - publish-to-pypi: - name: Publish to PyPI - needs: build - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/mars-ms - permissions: - id-token: write # Required for trusted publishing - - steps: - - name: Download distribution packages - uses: actions/download-artifact@v7 - with: - name: python-package-distributions - path: dist/ - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4530410..6983481 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,10 +1,23 @@ name: Tests +# The Python implementation is frozen to bug fixes and will be archived once the C# one +# has been used in earnest, but while the code is here its tests still run. Scoped to the +# Python tree so a C#-only change does not trigger them. on: push: branches: [main] + paths: + - 'mars/**' + - 'tests/**' + - 'pyproject.toml' + - '.github/workflows/tests.yml' pull_request: branches: [main] + paths: + - 'mars/**' + - 'tests/**' + - 'pyproject.toml' + - '.github/workflows/tests.yml' jobs: test: diff --git a/.gitignore b/.gitignore index 172241c..55d5e51 100644 --- a/.gitignore +++ b/.gitignore @@ -87,4 +87,11 @@ dmypy.json # Project-specific example-data/ -example-astral-data/ \ No newline at end of file +example-astral-data/ +# .NET +dotnet/**/bin/ +dotnet/**/obj/ +*.user +.mars-tmp/ +dotnet/**/TestResults/ +dotnet/TestResults/ diff --git a/CLAUDE.md b/CLAUDE.md index 66f10b5..3e8dd10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,13 @@ This file provides context and instructions for AI agents working on the Mars re ## Repository Overview -Mars (Mass Accuracy Recalibration System) is a tool for calibrating DIA mass spectrometry data from the Thermo Stellar instrument. It uses XGBoost to learn m/z corrections from spectral library matches. +MARS (Mass Accuracy Recalibration System) calibrates DIA mass spectrometry data, primarily +from the Thermo Stellar instrument. It learns the systematic part of the m/z error from +spectral library matches with gradient boosted trees, then subtracts it from every peak and +writes a corrected mzML. + +The shipping implementation is the C# one under `dotnet/`. The Python package in `mars/` is +frozen; see "MARS is the C# implementation" below. ## Continuous Integration (CI/CD) @@ -12,35 +18,107 @@ The repository uses GitHub Actions for CI/CD, defined in `.github/workflows/`. ### Workflows -1. **Tests (`tests.yml`)** - * **Triggers:** Push to `main`, Pull Requests to `main`. - * **Actions:** - * Sets up Python 3.10, 3.11, 3.12. - * Installs dependencies with `pip install -e ".[dev]"`. - * Runs tests using `pytest tests/ -v --tb=short`. - * Runs linting with `ruff check mars/`. - -2. **Publish to PyPI (`publish.yml`)** - * **Triggers:** Release published. - * **Actions:** - * Builds the package (`python -m build`). - * Publishes to PyPI using Trusted Publishing (OIDC). - * Requires the tag (e.g., `v0.1.0`) to match the release. +1. **.NET (`dotnet.yml`)** - the one that matters. Triggers on pushes and pull requests + touching `dotnet/`. + * Builds and tests on ubuntu, windows and macos with `-warnaserror`. + * Packages all six self-contained artifacts (`win-x64`, `win-arm64`, `linux-x64`, + `linux-arm64`, `osx-arm64`, `osx-x64`) on every run, so packaging breaks surface + here rather than at tag time. + * Builds and tests against `net10.0` as well, reported rather than gated. Pass + `-p:MarsIncludeNet10=true`, never a semicolon-separated framework list - the shell + eats the quotes and MSBuild misreads it. + * Runs **determinism** and the **vendored Osprey.ML drift guard** as separate jobs so + a failure in either is unmistakable. + +2. **.NET release (`dotnet-release.yml`)** - triggers on a `v*` tag. Preflights that the + tag, `` and the release notes agree before building anything, then publishes + the six artifacts and the GitHub Release. Also runs manually to build artifacts without + releasing. + +3. **Tests (`tests.yml`)** - the frozen Python package's pytest and ruff run, scoped to + `mars/`, `tests/` and `pyproject.toml` so a C#-only change does not trigger them. + +There is no PyPI workflow. The Python package is no longer released. ## Common Development Tasks -* **Install for dev:** `pip install -e ".[dev]"` -* **Run tests:** `pytest tests/` -* **Lint:** `ruff check .` -* **Build package:** `python -m build` +From `dotnet/`: + +* **Build:** `dotnet build -c Release -warnaserror` +* **Test:** `dotnet test -c Release` +* **Single-file binary:** `dotnet publish MARS/MARS.csproj -c Release -r + --self-contained true -p:PublishSingleFile=true` +* **Check against Python:** see `docs/python-parity.md` + +### Building with vendor support + +Reading Thermo, Bruker or Sciex data, and writing anything but mzML, needs +[pwiz-sharp](https://github.com/ProteoWizard/pwiz/pull/4178) - the .NET port of the +ProteoWizard core, still an unmerged draft with no package feed. The reference is optional: +without a checkout, `MARS_NO_PWIZ` drops that code and MARS reads and writes mzML exactly as +before, which is the configuration CI builds. Do not break that - a plain +`dotnet build`/`dotnet test` with no pwiz anywhere has to keep working. + +``` +dotnet build -c Release -p:PwizSharpDir=/pwiz/pwiz-sharp -p:IAgreeToVendorLicenses=true +``` + +Three things that are not obvious: + +* It needs the **full** pwiz working tree, not a sparse checkout of `pwiz-sharp/`. Bruker + reads its archives from `pwiz_aux` and pulls VC90 CRT files from `pwiz_tools/Shared/Lib`. +* pwiz-sharp needs a `global.json` pinning SDK 8, which is absent from the branch. Without + one, a nested `dotnet run` for its vendor pins generator fails to resolve an SDK. +* `dotnet/Directory.Build.rsp` carries a `WarningsNotAsErrors` that a single-file publish + needs. It repeats pwiz's own list because a global property replaces rather than extends + what a project sets; keep it in step with `pwiz-sharp/Directory.Build.props`. + +For the frozen Python package: `pip install -e ".[dev]"`, `pytest tests/`, `ruff check .` ## Release Notes -When making bug fixes or improvements during development: +One rolling draft, `release-notes/RELEASE_NOTES_next.md`. `release-notes/README.md` is the +authoritative process; the short version: + +* **Append to the draft as you land changes.** It is renamed to + `RELEASE_NOTES_v{version}.md` at release time and published verbatim as the GitHub + Release description, so write it for the people reading the Releases page. +* **Never edit a released notes file.** Versioned files record what shipped. +* **Be specific:** what was fixed and why, with numbers where they exist. +* **Group related changes:** New Features, Bug Fixes, Performance, Breaking Changes. +* **Flag anything that changes written output.** Corrected mzML files may already be in + downstream pipelines. + +## Documentation + +* `docs/` is the detailed documentation: the algorithm, the model and training, library + guidance, the mzML passthrough contract, the parity harness, and the port + specification. Update these when behaviour changes, not just the code comments. +* `README.md` is what a new user reads first and documents the C# tool. + `README-python.md` preserves the frozen Python implementation's documentation; leave it + as history rather than extending it. + +## MARS is the C# implementation + +The C# implementation under `dotnet/` **is** MARS. All new work goes there. + +The Python implementation is **frozen**: bug fixes only, no new features. It is no longer +published to PyPI and will be archived once the C# one has been used in earnest. Do not +add features to it, and do not port a Python quirk into C# without checking +`docs/dotnet-port-spec.md` section 10a first - four of them are known defects that C# +deliberately does not reproduce. + +Fragment matching and every model feature are verified against the Python implementation +row by row; see `docs/python-parity.md`. If you change the matcher or a feature, re-run +that comparison. A change that moves it is either a bug or a deliberate divergence that +belongs in section 10a - it is not something to accept quietly. + +## Versioning -* **Update the current release notes:** Add any fixes or changes to the current version's release notes file in `release-notes/` (e.g., `RELEASE_NOTES_v0.1.4.md`). -* **Be specific:** Document what was fixed and why. -* **Group related changes:** Use appropriate sections (Bug Fixes, Changes, New Features, etc.). +`YY.feature.patch`, starting at `26.1.0`. The version lives in exactly one place, +`dotnet/Directory.Build.props` (``), and changes only at release time. The +`0.1.x` line was the Python package. `release-notes/README.md` is the authoritative +process; pushing a `v{version}` tag builds every artifact and creates the GitHub Release. ## Style Guidelines diff --git a/README-python.md b/README-python.md new file mode 100644 index 0000000..1287cd6 --- /dev/null +++ b/README-python.md @@ -0,0 +1,304 @@ +# MARS: the Python implementation + +> [!NOTE] +> **This documents the frozen Python implementation.** MARS is now the C# tool; see the +> [main README](README.md). The Python package is bug-fix only, is no longer published to +> PyPI, and will be archived once the C# implementation has been used in earnest. It is +> kept here so existing users can still read how it works, and because its output is the +> reference the C# implementation is verified against +> ([docs/python-parity.md](docs/python-parity.md)). + +[![PyPI version](https://img.shields.io/pypi/v/mars-ms.svg)](https://pypi.org/project/mars-ms/) +[![Python versions](https://img.shields.io/pypi/pyversions/mars-ms.svg)](https://pypi.org/project/mars-ms/) +[![License](https://img.shields.io/pypi/l/mars-ms.svg)](https://github.com/maccoss/mars/blob/main/LICENSE) + +Mass recalibration tool for DIA mass spectrometry data from the ThermoFisher Stellar. + + +## Overview + +Mars learns m/z calibration corrections from spectral library fragment matches. The XGBoost model accounts for: + +- **Fragment m/z**: Mass-dependent calibration bias +- **Peak intensity**: Higher intensity peaks provide more reliable calibration +- **Absolute time**: Calibration drift over the acquisition run +- **Spectrum TIC**: Space charge effects from high ion current +- **Ion injection time**: Signal accumulation duration effects +- **Precursor m/z**: DIA isolation window-specific effects +- **RF temperatures**: Thermal effects from RF amplifier (RFA2) and electronics (RFC2) + +## How It Works + +1. **Fragment matching**: For each DIA MS2 spectrum, Mars finds library peptides where: + - The precursor m/z falls within the DIA isolation window + - The spectrum RT is within the peptide's elution window + +2. **Peak selection**: For each expected fragment, Mars selects the **most intense** peak within the m/z tolerance (not the closest), filtering for minimum intensity + +3. **Model training**: Each matched fragment becomes a training point with up to 16 features (see [Model Features](#model-features)) and target: `delta_mz` + +4. **Calibration**: The trained model predicts m/z corrections for all peaks in the mzML + +## Installation + +### From PyPI + +The last version published was `0.1.5`. No further releases will be made. + +```bash +pip install mars-ms +``` + +### From source (recommended, since it is the only way to get bug fixes) + +```bash +git clone https://github.com/maccoss/mars.git +cd mars +pip install -e . +``` + +**Requirements**: Python 3.10+, pyteomics, xgboost, numpy, pandas, matplotlib, seaborn, click + +## Usage + +### With PRISM CSV (Recommended) + +Use a CSV file created using this [Skyline report](Skyline-PRISM-Report/Skyline-PRISM.skyr) for accurate RT windows: + +```bash +mars calibrate \ + --mzML data.mzML \ + --prism-csv prism_report.csv \ + --tolerance 0.3 \ + --max-isolation-window 5.0 \ + --output-dir output/ +``` + +> **Note:** Both `--mzml` and `--mzML` are accepted. + +### With DIA-NN Parquet Output + +Use DIA-NN parquet files directly as a spectral library: + +```bash +mars calibrate \ + --mzml data.mzML \ + --library report-lib.parquet \ + --output-dir output/ +``` + +Mars automatically looks for `report.parquet` in the same directory to get RT windows. If the report file is in a different location: + +```bash +mars calibrate \ + --mzml data.mzML \ + --library report-lib.parquet \ + --diann-report /path/to/report.parquet \ + --output-dir output/ +``` + +### Basic Usage (blib) + +```bash +mars calibrate --mzml data.mzML --library library.blib --output-dir output/ +``` + +### Batch Processing + +```bash +# Multiple files with wildcard (no quotes needed) +mars calibrate --mzml *.mzML --library library.blib --output-dir output/ + +# Positional arguments also work (no --mzml flag needed) +mars calibrate *.mzML --library library.blib --output-dir output/ + +# Specify files individually +mars calibrate --mzml a.mzML --mzml b.mzML --library library.blib --output-dir output/ + +# All files in directory +mars calibrate --mzml-dir /path/to/data/ --library library.blib --output-dir output/ +``` + +### Applying a Pre-Trained Model + +If you've already trained a calibration model and want to apply it to new files without retraining: + +```bash +# Apply existing model to new mzML files +mars apply --mzml new_data.mzML --model mars_model.pkl --output-dir output/ + +# Apply to multiple files (no quotes needed) +mars apply --mzml *.mzML --model mars_model.pkl --output-dir output/ + +# Or as positional arguments +mars apply *.mzML --model mars_model.pkl --output-dir output/ + +# Apply to all files in a directory +mars apply --mzml-dir /path/to/data/ --model mars_model.pkl --output-dir output/ +``` + +This is useful when: + +- You want to calibrate files from the same instrument/method without retraining +- You trained on a subset of files and want to apply to the rest +- You're reprocessing data with a validated model + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--mzml` / `--mzML` | - | Path to mzML file(s) or glob pattern (repeatable) | +| `--mzml-dir` | - | Directory containing mzML files | +| `--library` | - | Path to spectral library: blib file or DIA-NN `report-lib.parquet` | +| `--prism-csv` | - | PRISM Skyline CSV with Start/End Time columns | +| `--diann-report` | - | Path to DIA-NN `report.parquet` (auto-detected if in same dir as library) | +| `--tolerance` | 0.3 | m/z tolerance for matching (Th), ignored if `--tolerance-ppm` is set | +| `--tolerance-ppm` | - | m/z tolerance for matching in ppm (e.g., 10 for Astral), overrides `--tolerance` | +| `--min-intensity` | 500 | Minimum peak intensity for matching | +| `--max-isolation-window` | - | Maximum isolation window width (m/z) to include | +| `--temperature-dir` | - | Directory with RF temperature CSV files | +| `--output-dir` | `.` | Output directory | +| `--model-path` | - | Path to save/load calibration model | +| `--no-recalibrate` | - | Only train model, don't write mzML | + +## RT Window Behavior + +- **With `--prism-csv`**: Uses exact `Start Time` and `End Time` from Skyline +- **With DIA-NN parquet**: Uses `RT.Start` and `RT.Stop` from `report.parquet` +- **With blib only**: Uses +/-5 seconds around the blib library RT + +## Isolation Window Filtering + +Some DIA methods use wide isolation windows (e.g., 20-30 m/z) that may reduce calibration accuracy. Use `--max-isolation-window` to exclude these: + +```bash +# Exclude windows wider than 5 m/z +mars calibrate --mzml data.mzML --prism-csv report.csv --max-isolation-window 5.0 +``` + +This filters spectra during both model training and mzML recalibration. Typical narrow DIA windows (~1 m/z) are retained. + +## Output Files + +| File | Description | +|------|-------------| +| `{input}-mars.mzML` | Recalibrated mzML file | +| `mars_model.pkl` | Trained XGBoost calibration model | +| `mars_qc_histogram.png` | Delta m/z distribution (before/after) | +| `mars_qc_heatmap.png` | 2D heatmap (RT × m/z, color = delta) | +| `mars_qc_intensity_vs_error.png` | Intensity vs mass error hexbin | +| `mars_qc_rt_vs_error.png` | RT vs mass error hexbin | +| `mars_qc_mz_vs_error.png` | Fragment m/z vs mass error hexbin | +| `mars_qc_tic_vs_error.png` | TIC vs mass error hexbin | +| `mars_qc_injection_time_vs_error.png` | Injection time vs mass error hexbin | +| `mars_qc_tic_injection_time_vs_error.png` | TIC×injection time vs mass error hexbin | +| `mars_qc_fragment_ions_vs_error.png` | Fragment ions vs mass error hexbin | +| `mars_qc_rfa2_temperature_vs_error.png` | RFA2 temperature vs error (if available) | +| `mars_qc_rfc2_temperature_vs_error.png` | RFC2 temperature vs error (if available) | +| `mars_qc_feature_importance.png` | Model feature importance | +| `mars_qc_summary.txt` | Calibration statistics | + + +## Model Features + +The XGBoost model uses up to 16 features to predict m/z corrections: + +1. `precursor_mz` - DIA isolation window center +2. `fragment_mz` - Fragment m/z being calibrated +3. `absolute_time` - Time relative to first acquisition (seconds) +4. `log_tic` - Log10 of spectrum total ion current +5. `log_intensity` - Log10 of peak intensity +6. `injection_time` - Ion injection time (seconds) +7. `tic_injection_time` - TIC × injection time product +8. `fragment_ions` - Fragment intensity × injection time (total ions, not rate) +9. `ions_above_0_1` - Total ions in (X+0.5, X+1.5] Th range above fragment m/z +10. `ions_above_1_2` - Total ions in (X+1.5, X+2.5] Th range above fragment m/z +11. `ions_above_2_3` - Total ions in (X+2.5, X+3.5] Th range above fragment m/z +12. `ions_below_0_1` - Total ions in (X-1.5, X-0.5] Th range below fragment m/z +13. `ions_below_1_2` - Total ions in (X-2.5, X-1.5] Th range below fragment m/z +14. `ions_below_2_3` - Total ions in (X-3.5, X-2.5] Th range below fragment m/z +15. `adjacent_ratio_0_1` - ions_above_0_1 / fragment_ions (relative adjacent density) +16. `adjacent_ratio_1_2` - ions_above_1_2 / fragment_ions +17. `adjacent_ratio_2_3` - ions_above_2_3 / fragment_ions +18. `adjacent_ratio_below_0_1` - ions_below_0_1 / fragment_ions +19. `adjacent_ratio_below_1_2` - ions_below_1_2 / fragment_ions +20. `adjacent_ratio_below_2_3` - ions_below_2_3 / fragment_ions +21. `rfa2_temp` - RF amplifier temperature (°C) +22. `rfc2_temp` - RF electronics temperature (°C) + +**Note**: Features 6-20 are only included if injection time data is available in the mzML files. Features 21-22 are only included if temperature CSV files are provided. Features with universally missing data are automatically excluded. + +## RF Temperature Data + +Mars can incorporate RF temperature data to model thermal effects on mass accuracy. Temperature data is loaded from CSV files exported from Thermo chromatogram exports. + +### Temperature File Format + +Temperature CSV files should be in Thermo's chromatogram export format: +- 3 header lines (skipped) +- Columns: `Time(min)`, temperature value + +Example naming convention: +``` +RFA2-Sample_Name.csv # RF amplifier temperature +RFC2-Sample_Name.csv # RF electronics temperature +``` + +### Usage with Temperature Data + +```bash +mars calibrate \ + --mzml data.mzML \ + --prism-csv report.csv \ + --temperature-dir /path/to/temperature_csvs/ \ + --output-dir output/ +``` + +Mars automatically finds temperature files matching each mzML filename and interpolates temperature values at each spectrum's retention time. + +## Python API + +```python +from mars import load_blib, read_dia_spectra, match_library_to_spectra, MzCalibrator + +# Load library and match +library = load_blib("library.blib") +spectra = read_dia_spectra("data.mzML") +matches = match_library_to_spectra(library, spectra, mz_tolerance=0.2, min_intensity=1500) + +# Train and save model +calibrator = MzCalibrator() +calibrator.fit(matches) +calibrator.save("model.pkl") +``` + +### Using DIA-NN Parquet + +```python +from mars import load_diann_library, read_dia_spectra, match_library_to_spectra, MzCalibrator + +# Load DIA-NN library (auto-finds report.parquet in same directory) +library = load_diann_library("report-lib.parquet") + +# Or specify report.parquet explicitly +library = load_diann_library("report-lib.parquet", report_parquet="/path/to/report.parquet") + +# Filter to specific mzML file(s) +library = load_diann_library("report-lib.parquet", mzml_filename=["sample1.mzML", "sample2.mzML"]) + +spectra = read_dia_spectra("data.mzML") +matches = match_library_to_spectra(library, spectra, mz_tolerance=0.2, min_intensity=1500) +``` + +## Requirements + +- **Spectral library**: One of the following formats: + - blib format from Skyline with fragment annotations + - DIA-NN parquet output (`report-lib.parquet` + `report.parquet`) +- **mzML files**: DIA data from Thermo Stellar (or similar unit resolution instrument) +- **PRISM CSV** (optional): Skyline report with `Start Time`, `End Time`, `Replicate Name` columns + +## License + +MIT + diff --git a/README.md b/README.md index 55286d9..fb77cba 100644 --- a/README.md +++ b/README.md @@ -1,293 +1,368 @@ # MARS: Mass Accuracy Recalibration System -[![PyPI version](https://img.shields.io/pypi/v/mars-ms.svg)](https://pypi.org/project/mars-ms/) -[![Python versions](https://img.shields.io/pypi/pyversions/mars-ms.svg)](https://pypi.org/project/mars-ms/) -[![License](https://img.shields.io/pypi/l/mars-ms.svg)](https://github.com/maccoss/mars/blob/main/LICENSE) +[![Build](https://github.com/maccoss/mars/actions/workflows/dotnet.yml/badge.svg)](https://github.com/maccoss/mars/actions/workflows/dotnet.yml) +[![Release](https://img.shields.io/github/v/release/maccoss/mars?display_name=tag&sort=semver&label=release)](https://github.com/maccoss/mars/releases/latest) +[![.NET](https://img.shields.io/badge/.NET-8.0%20%7C%2010.0-512BD4)](https://dotnet.microsoft.com/download) +[![License](https://img.shields.io/github/license/maccoss/mars)](https://github.com/maccoss/mars/blob/main/LICENSE) -Mass recalibration tool for DIA mass spectrometry data from the ThermoFisher Stellar. +Learns the systematic part of a mass spectrometer's m/z error from spectral library +matches, and subtracts it from every peak in the file. -## Overview +Reads Thermo, Bruker and Sciex data directly as well as mzML, and writes mzML, mzXML, mzMLb or +mgf - so a run can be calibrated straight off the instrument with no conversion step. -Mars learns m/z calibration corrections from spectral library fragment matches. The XGBoost model accounts for: +On Thermo Stellar ion-trap DIA data this cuts the median absolute fragment mass error +roughly in half: -- **Fragment m/z**: Mass-dependent calibration bias -- **Peak intensity**: Higher intensity peaks provide more reliable calibration -- **Absolute time**: Calibration drift over the acquisition run -- **Spectrum TIC**: Space charge effects from high ion current -- **Ion injection time**: Signal accumulation duration effects -- **Precursor m/z**: DIA isolation window-specific effects -- **RF temperatures**: Thermal effects from RF amplifier (RFA2) and electronics (RFC2) +| Stellar HeLa GPF-DIA, 5 files | Uncorrected | Corrected | +|---|---|---| +| Median absolute deviation | 0.0800 Th | **0.0464 Th** | +| Standard deviation | 0.1180 Th | **0.0872 Th** | +| Median error | -0.0082 Th | **-0.0025 Th** | -## How It Works +Measured by rematching the library against the written output. On an already +well-calibrated Astral run the same pipeline moves the spread by under 2% - there is +little systematic error left to remove. **Run `mars qc` first** to see whether your data +has anything worth correcting. -1. **Fragment matching**: For each DIA MS2 spectrum, Mars finds library peptides where: - - The precursor m/z falls within the DIA isolation window - - The spectrum RT is within the peptide's elution window +## About the Python implementation -2. **Peak selection**: For each expected fragment, Mars selects the **most intense** peak within the m/z tolerance (not the closest), filtering for minimum intensity +MARS began as a Python package (`mars-ms`, versions `0.1.x`). **The C# tool documented +here is MARS going forward.** The Python implementation is frozen to bug fixes, is no +longer published to PyPI, and will be archived once the C# one has been used in earnest. +Its documentation is preserved in [README-python.md](README-python.md). -3. **Model training**: Each matched fragment becomes a training point with up to 16 features (see [Model Features](#model-features)) and target: `delta_mz` +The C# implementation is not a rewrite that hopes to behave the same. Fragment matching +and every model feature were verified against the Python implementation row by row: across +160,947 matched fragments from two Stellar runs, all 24 shared columns agree with a maximum +absolute difference of **zero**. See [docs/python-parity.md](docs/python-parity.md). -4. **Calibration**: The trained model predicts m/z corrections for all peaks in the mzML +Where they deliberately differ, it is because the port found four defects in the Python +implementation - including an invalid SHA-1 checksum on every mzML it has ever written. +Those are listed in +[docs/dotnet-port-spec.md](docs/dotnet-port-spec.md#10a-defects-found-in-the-python-implementation). -## Installation +Versions follow `YY.feature.patch` starting at `26.1.0`; the `0.1.x` line was the Python +package. See [release-notes/README.md](release-notes/README.md). -### From PyPI (recommended) +--- -```bash -pip install mars-ms -``` +## Install + +### Option 1: download a build (no .NET needed) -### From source +Grab an archive from the [Releases page](https://github.com/maccoss/mars/releases). Each is +self-contained - unpack it and run; nothing else to install. + +| Platform | Archive | +|---|---| +| Windows x64 | `mars-{version}-win-x64.zip` | +| Windows on Arm | `mars-{version}-win-arm64.zip` | +| Linux x64 | `mars-{version}-linux-x64.tar.gz` | +| Linux arm64 | `mars-{version}-linux-arm64.tar.gz` | +| macOS Apple silicon | `mars-{version}-osx-arm64.tar.gz` | +| macOS Intel | `mars-{version}-osx-x64.tar.gz` | ```bash -git clone https://github.com/maccoss/mars.git -cd mars -pip install -e . +tar xzf mars-26.1.0-linux-x64.tar.gz +./mars --version ``` -**Requirements**: Python 3.10+, pyteomics, xgboost, numpy, pandas, matplotlib, seaborn, click +`SHA256SUMS.txt` is published alongside. On macOS, Gatekeeper will quarantine an +unsigned download; clear it with `xattr -d com.apple.quarantine mars` or allow it once +under System Settings > Privacy & Security. -## Usage +Builds of unreleased commits are available as workflow artifacts on any CI run, under the +Actions tab. -### With PRISM CSV (Recommended) +### Option 2: build a single binary yourself -Use a CSV file created using this [Skyline report](Skyline-PRISM-Report/Skyline-PRISM.skyr) for accurate RT windows: +Build once (see [Build from source](#build-from-source)) and copy the resulting file +anywhere; it carries its own runtime. ```bash -mars calibrate \ - --mzML data.mzML \ - --prism-csv prism_report.csv \ - --tolerance 0.3 \ - --max-isolation-window 5.0 \ - --output-dir output/ +# from the dotnet/ directory, pick the target you want +dotnet publish MARS/MARS.csproj -c Release -f net8.0 \ + -r win-x64 --self-contained true -p:PublishSingleFile=true -o publish/win-x64 +dotnet publish MARS/MARS.csproj -c Release -f net8.0 \ + -r linux-x64 --self-contained true -p:PublishSingleFile=true -o publish/linux-x64 +dotnet publish MARS/MARS.csproj -c Release -f net8.0 \ + -r osx-arm64 --self-contained true -p:PublishSingleFile=true -o publish/osx-arm64 ``` -> **Note:** Both `--mzml` and `--mzML` are accepted. +Produces a ~69 MB `mars` (`mars.exe` on Windows). Other runtime identifiers: +`linux-arm64`, `osx-x64`. -### With DIA-NN Parquet Output +### Option 3: install the .NET runtime -Use DIA-NN parquet files directly as a spectral library: +A framework-dependent build is about 2 MB but needs the **.NET 8 runtime** (or newer - MARS +rolls forward). To *build* MARS you need the **.NET 8 SDK**, which includes the runtime. -```bash -mars calibrate \ - --mzml data.mzML \ - --library report-lib.parquet \ - --output-dir output/ +**Windows** + +```powershell +winget install Microsoft.DotNet.SDK.8 +# runtime only: +winget install Microsoft.DotNet.Runtime.8 ``` -Mars automatically looks for `report.parquet` in the same directory to get RT windows. If the report file is in a different location: +Or download from [dot.net/download](https://dotnet.microsoft.com/download/dotnet/8.0). + +**Linux** ```bash -mars calibrate \ - --mzml data.mzML \ - --library report-lib.parquet \ - --diann-report /path/to/report.parquet \ - --output-dir output/ -``` +# Ubuntu 22.04+ / Debian 12+ +sudo apt update && sudo apt install -y dotnet-sdk-8.0 -### Basic Usage (blib) +# Fedora / RHEL +sudo dnf install -y dotnet-sdk-8.0 -```bash -mars calibrate --mzml data.mzML --library library.blib --output-dir output/ +# any distro, no root required +curl -sSL https://dot.net/v1/dotnet-install.sh | bash -s -- --channel 8.0 +export PATH="$HOME/.dotnet:$PATH" ``` -### Batch Processing +**macOS** ```bash -# Multiple files with wildcard (no quotes needed) -mars calibrate --mzml *.mzML --library library.blib --output-dir output/ +brew install --cask dotnet-sdk +``` -# Positional arguments also work (no --mzml flag needed) -mars calibrate *.mzML --library library.blib --output-dir output/ +Or download the `.pkg` from +[dot.net/download](https://dotnet.microsoft.com/download/dotnet/8.0). On Apple silicon +take the **Arm64** build. -# Specify files individually -mars calibrate --mzml a.mzML --mzml b.mzML --library library.blib --output-dir output/ +Check it worked: -# All files in directory -mars calibrate --mzml-dir /path/to/data/ --library library.blib --output-dir output/ +```bash +dotnet --list-sdks ``` -### Applying a Pre-Trained Model - -If you've already trained a calibration model and want to apply it to new files without retraining: +## Build from source ```bash -# Apply existing model to new mzML files -mars apply --mzml new_data.mzML --model mars_model.pkl --output-dir output/ +git clone https://github.com/maccoss/mars.git +cd mars/dotnet +dotnet build -c Release +dotnet test +``` -# Apply to multiple files (no quotes needed) -mars apply --mzml *.mzML --model mars_model.pkl --output-dir output/ +The CLI lands at `MARS/bin/Release/net8.0/mars` (`mars.exe` on Windows). Add that +directory to your `PATH`, or publish a single binary as above. -# Or as positional arguments -mars apply *.mzML --model mars_model.pkl --output-dir output/ +**Dependencies are handled by `dotnet build`.** There is exactly one NuGet package, +`Parquet.Net`, used to read DIA-NN libraries. It brings a small native compression library +(`nironcompress`) that ships alongside the binary in every release archive - nothing has to +be installed separately, but it does mean release artifacts are per-platform. -# Apply to all files in a directory -mars apply --mzml-dir /path/to/data/ --model mars_model.pkl --output-dir output/ -``` +`.blib` files are read through a managed SQLite reader written for this purpose rather than +`Microsoft.Data.Sqlite`, so that path adds no native code. `MARS.Core` and `MARS.OspreyML` +have no package references at all and are pure managed; only `MARS.IO` pulls in +`Parquet.Net`. -This is useful when: +Targets `net8.0` by default, which runs unchanged on .NET 9 and 10. With a .NET 10 SDK +installed you can build the full matrix: -- You want to calibrate files from the same instrument/method without retraining -- You trained on a subset of files and want to apply to the rest -- You're reprocessing data with a validated model +```bash +dotnet build -c Release -p:MarsIncludeNet10=true +``` -## Options +### Platform status -| Option | Default | Description | -|--------|---------|-------------| -| `--mzml` / `--mzML` | - | Path to mzML file(s) or glob pattern (repeatable) | -| `--mzml-dir` | - | Directory containing mzML files | -| `--library` | - | Path to spectral library: blib file or DIA-NN `report-lib.parquet` | -| `--prism-csv` | - | PRISM Skyline CSV with Start/End Time columns | -| `--diann-report` | - | Path to DIA-NN `report.parquet` (auto-detected if in same dir as library) | -| `--tolerance` | 0.3 | m/z tolerance for matching (Th), ignored if `--tolerance-ppm` is set | -| `--tolerance-ppm` | - | m/z tolerance for matching in ppm (e.g., 10 for Astral), overrides `--tolerance` | -| `--min-intensity` | 500 | Minimum peak intensity for matching | -| `--max-isolation-window` | - | Maximum isolation window width (m/z) to include | -| `--temperature-dir` | - | Directory with RF temperature CSV files | -| `--output-dir` | `.` | Output directory | -| `--model-path` | - | Path to save/load calibration model | -| `--no-recalibrate` | - | Only train model, don't write mzML | +CI builds and tests on Windows, Linux and macOS, and packages all six runtime +identifiers on every push. -## RT Window Behavior +| Platform | Status | +|---|---| +| Windows x64 | Build, tests and packaging in CI; full pipeline run on 6 GB of real data | +| Linux x64 | Build, tests and packaging in CI; verified locally on real mzML | +| macOS arm64 / x64 | Build, tests and packaging in CI. **Not yet run on real data** by a human. | +| Windows on Arm | Cross-compiled and packaged; binary confirmed ARM64, **not** smoke tested - no hosted Windows Arm runner | +| Linux arm64 | Cross-compiled and packaged, **not** smoke tested - no hosted arm64 Linux runner | -- **With `--prism-csv`**: Uses exact `Start Time` and `End Time` from Skyline -- **With DIA-NN parquet**: Uses `RT.Start` and `RT.Stop` from `report.parquet` -- **With blib only**: Uses +/-5 seconds around the blib library RT +Vendor reading and the non-mzML outputs are a separate axis, because they need a build made +against pwiz-sharp. All six runtime identifiers publish and run with them; the gaps are that +**Sciex is Windows-only**, because its SDK is, and **mzMLb is x64-only**, because the HDF5 +library it needs is published for x64 alone. Thermo and Bruker reading, and mzML and mzXML +writing, work on every target. None of it is exercised in CI yet - pwiz-sharp has no package +feed, so CI builds the mzML-only configuration. -## Isolation Window Filtering +> **On Windows on Arm and macOS Intel**, `Parquet.Net`'s native compression library is not +> published for the platform, so a DIA-NN library compressed with **LZ4 or LZO** cannot be +> read there and fails with a clear "no compression codec" message. Snappy (parquet's usual +> default, and what DIA-NN writes), Gzip, Brotli, Zstd and uncompressed all work, as do +> Skyline PRISM reports and `.blib` libraries. Everything else in MARS is unaffected. + +## Usage -Some DIA methods use wide isolation windows (e.g., 20-30 m/z) that may reduce calibration accuracy. Use `--max-isolation-window` to exclude these: +The command is identical on all three platforms; only the path to the binary differs. ```bash -# Exclude windows wider than 5 m/z -mars calibrate --mzml data.mzML --prism-csv report.csv --max-isolation-window 5.0 -``` +# Linux / macOS +./mars calibrate --mzml-dir runs/ --prism-csv report.csv --output-dir corrected/ -This filters spectra during both model training and mzML recalibration. Typical narrow DIA windows (~1 m/z) are retained. - -## Output Files - -| File | Description | -|------|-------------| -| `{input}-mars.mzML` | Recalibrated mzML file | -| `mars_model.pkl` | Trained XGBoost calibration model | -| `mars_qc_histogram.png` | Delta m/z distribution (before/after) | -| `mars_qc_heatmap.png` | 2D heatmap (RT × m/z, color = delta) | -| `mars_qc_intensity_vs_error.png` | Intensity vs mass error hexbin | -| `mars_qc_rt_vs_error.png` | RT vs mass error hexbin | -| `mars_qc_mz_vs_error.png` | Fragment m/z vs mass error hexbin | -| `mars_qc_tic_vs_error.png` | TIC vs mass error hexbin | -| `mars_qc_injection_time_vs_error.png` | Injection time vs mass error hexbin | -| `mars_qc_tic_injection_time_vs_error.png` | TIC×injection time vs mass error hexbin | -| `mars_qc_fragment_ions_vs_error.png` | Fragment ions vs mass error hexbin | -| `mars_qc_rfa2_temperature_vs_error.png` | RFA2 temperature vs error (if available) | -| `mars_qc_rfc2_temperature_vs_error.png` | RFC2 temperature vs error (if available) | -| `mars_qc_feature_importance.png` | Model feature importance | -| `mars_qc_summary.txt` | Calibration statistics | - - -## Model Features - -The XGBoost model uses up to 16 features to predict m/z corrections: - -1. `precursor_mz` - DIA isolation window center -2. `fragment_mz` - Fragment m/z being calibrated -3. `absolute_time` - Time relative to first acquisition (seconds) -4. `log_tic` - Log10 of spectrum total ion current -5. `log_intensity` - Log10 of peak intensity -6. `injection_time` - Ion injection time (seconds) -7. `tic_injection_time` - TIC × injection time product -8. `fragment_ions` - Fragment intensity × injection time (total ions, not rate) -9. `ions_above_0_1` - Total ions in (X+0.5, X+1.5] Th range above fragment m/z -10. `ions_above_1_2` - Total ions in (X+1.5, X+2.5] Th range above fragment m/z -11. `ions_above_2_3` - Total ions in (X+2.5, X+3.5] Th range above fragment m/z -12. `ions_below_0_1` - Total ions in (X-1.5, X-0.5] Th range below fragment m/z -13. `ions_below_1_2` - Total ions in (X-2.5, X-1.5] Th range below fragment m/z -14. `ions_below_2_3` - Total ions in (X-3.5, X-2.5] Th range below fragment m/z -15. `adjacent_ratio_0_1` - ions_above_0_1 / fragment_ions (relative adjacent density) -16. `adjacent_ratio_1_2` - ions_above_1_2 / fragment_ions -17. `adjacent_ratio_2_3` - ions_above_2_3 / fragment_ions -18. `adjacent_ratio_below_0_1` - ions_below_0_1 / fragment_ions -19. `adjacent_ratio_below_1_2` - ions_below_1_2 / fragment_ions -20. `adjacent_ratio_below_2_3` - ions_below_2_3 / fragment_ions -21. `rfa2_temp` - RF amplifier temperature (°C) -22. `rfc2_temp` - RF electronics temperature (°C) - -**Note**: Features 6-20 are only included if injection time data is available in the mzML files. Features 21-22 are only included if temperature CSV files are provided. Features with universally missing data are automatically excluded. - -## RF Temperature Data - -Mars can incorporate RF temperature data to model thermal effects on mass accuracy. Temperature data is loaded from CSV files exported from Thermo chromatogram exports. - -### Temperature File Format - -Temperature CSV files should be in Thermo's chromatogram export format: -- 3 header lines (skipped) -- Columns: `Time(min)`, temperature value - -Example naming convention: +# Windows (PowerShell) +.\mars.exe calibrate --mzml-dir runs\ --prism-csv report.csv --output-dir corrected\ ``` -RFA2-Sample_Name.csv # RF amplifier temperature -RFC2-Sample_Name.csv # RF electronics temperature + +### Before you correct anything + +```bash +mars qc --mzml-dir runs/ --prism-csv report.csv ``` -### Usage with Temperature Data +Reports the mass error already present, in both Th and ppm, without training or writing +anything. If the error is already small, MARS has nothing to offer and you have learned +that cheaply. + +### Correcting a set of runs ```bash mars calibrate \ - --mzml data.mzML \ - --prism-csv report.csv \ - --temperature-dir /path/to/temperature_csvs/ \ - --output-dir output/ + --mzml-dir runs/ \ + --prism-csv skyline-report.csv \ + --output-dir corrected/ ``` -Mars automatically finds temperature files matching each mzML filename and interpolates temperature values at each spectrum's retention time. +The inputs do not have to be mzML: a directory of Thermo `.raw`, or a Bruker `.d`, works the +same way, and `--output-format mzXML|mzMLb|mgf` picks what comes out. -## Python API +Writes `{input}-mars.mzML` for each input, plus `mars_model.json`, +`mars_qc_summary.txt` and `mars_qc_report.html`. All input files are fitted together as one +cohort, which is what lets the model learn drift across a run sequence rather than only +within a file. -```python -from mars import load_blib, read_dia_spectra, match_library_to_spectra, MzCalibrator +`mars_qc_report.html` is the one to look at: the error distribution before and after, the +error across retention time and m/z, feature importance, and a panel per feature. It is a +single self-contained file with everything embedded, so it can be emailed as an attachment +and read by someone who has neither the data nor the tool. See +[docs/qc-report.md](docs/qc-report.md), or pass `--no-html-report` to skip it. -# Load library and match -library = load_blib("library.blib") -spectra = read_dia_spectra("data.mzML") -matches = match_library_to_spectra(library, spectra, mz_tolerance=0.2, min_intensity=1500) +For high-resolution data use a relative tolerance: -# Train and save model -calibrator = MzCalibrator() -calibrator.fit(matches) -calibrator.save("model.pkl") +```bash +mars calibrate --mzml-dir runs/ --prism-csv report.csv \ + --tolerance-ppm 10 --output-dir corrected/ ``` -### Using DIA-NN Parquet - -```python -from mars import load_diann_library, read_dia_spectra, match_library_to_spectra, MzCalibrator +### Reusing a model -# Load DIA-NN library (auto-finds report.parquet in same directory) -library = load_diann_library("report-lib.parquet") - -# Or specify report.parquet explicitly -library = load_diann_library("report-lib.parquet", report_parquet="/path/to/report.parquet") +```bash +mars apply --model corrected/mars_model.json --mzml-dir more-runs/ \ + --output-dir corrected/ --validate +``` -# Filter to specific mzML file(s) -library = load_diann_library("report-lib.parquet", mzml_filename=["sample1.mzML", "sample2.mzML"]) +### Checking the file format is handled correctly -spectra = read_dia_spectra("data.mzML") -matches = match_library_to_spectra(library, spectra, mz_tolerance=0.2, min_intensity=1500) +```bash +mars verify runs/one.mzML ``` -## Requirements - -- **Spectral library**: One of the following formats: - - blib format from Skyline with fragment annotations - - DIA-NN parquet output (`report-lib.parquet` + `report.parquet`) -- **mzML files**: DIA data from Thermo Stellar (or similar unit resolution instrument) -- **PRISM CSV** (optional): Skyline report with `Start Time`, `End Time`, `Replicate Name` columns +Round-trips the file applying a **null correction**, then checks the result decodes to +bit-identical m/z and intensity arrays with a valid index and checksum. Run this first if +a corrected file misbehaves downstream: it separates a file-format problem from a model +problem, and those have very different fixes. + +### Input and output formats + +| Read | Vendor | Platforms | +|---|---|---| +| `.mzML` | - | everywhere, by MARS itself | +| `.raw` | Thermo | Windows, Linux, macOS | +| `.d`, `.tdf`, `.tsf`, `.baf` | Bruker | Windows, Linux | +| `.wiff`, `.wiff2` | Sciex | Windows | + +| Write | Notes | +|---|---| +| `mzML` (default) | The input byte for byte, except the m/z arrays that changed | +| `mzXML` | Cannot express ion mobility or some isolation-window terms | +| `mzMLb` | mzML in HDF5; roughly half the size. x64 only | +| `mgf` | MS2 peak lists only - no MS1, no chromatograms, no scan metadata | + +Vendor formats and the non-mzML outputs come from +[pwiz-sharp](https://github.com/ProteoWizard/pwiz/pull/4178), the .NET port of the ProteoWizard +core. Released binaries carry the vendor SDKs, so a download opens a `.raw` with nothing else +installed; `mars --version` reports what the binary in front of you actually has. A MARS built +without pwiz reads and writes mzML exactly as before, and says so when asked for anything else +- see [the CLI reference](docs/cli-reference.md#input-formats). + +Once pwiz-sharp merges upstream, MARS will stop shipping its own copies on Windows and use the +SDKs an installed Skyline-daily, Skyline or msconvert already provides - in that order, and +only after checking the version it finds. See +[open-questions.md](docs/open-questions.md#where-the-vendor-sdks-come-from). + +MARS reads the mass analyzer from the file and configures itself: 0.3 Th on a trap, 10 ppm on +an orbitrap, TOF or Astral, with the QC report drawn in matching units. `--resolution` and +`--tolerance` override it. On a timsTOF the ion mobility dimension is collapsed - each frame's +mobility scans are combined into one spectrum per isolation window - because MARS models m/z +error, not mobility. + +### Commands + +| Command | Purpose | +|---|---| +| `calibrate` | Learn a correction from library matches and write recalibrated output | +| `apply` | Apply an existing model to more files | +| `qc` | Report mass accuracy without training or writing | +| `verify` | Round-trip a file with a null correction and check it | +| `compare` | Compare two mzML files on decoded values | + +Every command takes `--help`. Diagnostics go to stderr so stdout stays pipeable. Exit +codes: `0` success, `1` input error, `2` insufficient training data, `3` output validation +failure. + +### Frequently used options + +| Option | Default | Meaning | +|---|---|---| +| `--mzml`, `--mzml-dir` | - | Input files, a glob, or a directory | +| `--prism-csv` | - | Skyline PRISM report (recommended library source) | +| `--library` | - | `.blib`, DIA-NN `report-lib.parquet`, or a PRISM `.csv` | +| `--diann-report` | beside the library | DIA-NN `report.parquet`, for RT windows | +| `--tolerance` | 0.3 Th | Matching tolerance; use `--tolerance-ppm` for Orbitrap/Astral | +| `--min-intensity` | 500 | Minimum peak intensity usable as a training row | +| `--max-isolation-window` | - | Leave wider isolation windows uncorrected | +| `--temperature-dir` | - | RF temperature logs (`RFA2-*.csv`, `RFC2-*.csv`) | +| `--threads` | `auto` | Worker threads; `auto` is one per logical processor, and the run reports which it used | +| `--on-reorder` | `clamp` | What to do if a correction would unsort the m/z array | + +## Which library do I need? + +One with **theoretical** fragment m/z. A [Skyline PRISM +report](Skyline-PRISM-Report/Skyline-PRISM.skyr) is the best-supported source. A `.blib` +without peak annotations cannot be used and MARS will say so rather than produce a bad +model. See [docs/spectral-libraries.md](docs/spectral-libraries.md). + +## A note on reproducibility + +Identical input produces identical decoded m/z values, on any thread count and any +platform. Compressed **file bytes** are not portable across platforms, because runtimes +ship different zlib builds - the same input produced 1,176,380 bytes on Windows and +1,176,172 on Linux with every decoded value identical. Compare files with `mars compare`, +not `cmp`. See [docs/algorithm.md](docs/algorithm.md#determinism). + +**mzMLb is the exception**, and not because of anything MARS does: two mzMLb writes of identical +data differ byte-wise, because the HDF5 container records things that vary between writes. The +spectra are the same. Use mzML or mzXML where a checksum has to match. + +## Documentation + +[docs/](docs/) is the full documentation. The pages most people want: + +| | | +|---|---| +| [docs/algorithm.md](docs/algorithm.md) | How the recalibration works: matching, features, model, correction | +| [docs/cli-reference.md](docs/cli-reference.md) | Every command and option, and what the exit codes mean | +| [docs/spectral-libraries.md](docs/spectral-libraries.md) | Library sources and choosing a tolerance | +| [docs/qc-report.md](docs/qc-report.md) | How to read the QC figures | +| [docs/model.md](docs/model.md) | The gradient boosted trees, in depth | +| [docs/mzml-passthrough.md](docs/mzml-passthrough.md) | How output files are written | +| [docs/architecture.md](docs/architecture.md) | A map of the code, for anyone modifying it | +| [docs/python-parity.md](docs/python-parity.md) | How this is verified against the Python implementation | +| [docs/dotnet-port-spec.md](docs/dotnet-port-spec.md) | Port specification, acceptance gates, measured results | +| [dotnet/README.md](dotnet/README.md) | The C# source tree | +| [release-notes/](release-notes/) | Per-version release notes and the release process | ## License -MIT - +MIT. See [LICENSE](LICENSE). diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..25aeee5 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,73 @@ +# MARS documentation + +## Start here + +| Document | What it covers | +|---|---| +| [algorithm.md](algorithm.md) | The recalibration algorithm end to end: fragment matching, the 22 features, training, and how the correction is applied. | +| [cli-reference.md](cli-reference.md) | Every command and option, and what the exit codes mean. | +| [spectral-libraries.md](spectral-libraries.md) | The four library sources, what makes a usable one, and how to choose a tolerance. | +| [qc-report.md](qc-report.md) | How to read the QC figures, and what a small correction actually means. | + +## In depth + +| Document | What it covers | +|---|---| +| [model.md](model.md) | The gradient boosted trees: objective, histogram splits, hyperparameters, intensity weighting, determinism, importance, and the model file format. | +| [mzml-passthrough.md](mzml-passthrough.md) | How MARS writes mzML without disturbing anything it did not mean to change. | +| [architecture.md](architecture.md) | A map of the code: projects, data flow, the managed SQLite reader, dependencies, and where to change things. | + +## Provenance + +| Document | What it covers | +|---|---| +| [python-parity.md](python-parity.md) | How the C# implementation is checked against the Python one row by row, what agrees, and what parity cannot cover. | +| [dotnet-port-spec.md](dotnet-port-spec.md) | The specification governing the port: decisions, acceptance gates, measured results, and four defects the port found in the Python implementation. | +| [open-questions.md](open-questions.md) | What was deliberately left undone, what was measured, and what would settle it. | + +--- + +## Quick answers + +**Should I run MARS on my data?** +Run `mars qc` first. It reports the mass error already present without training or writing +anything. On a well-calibrated instrument there is often nothing worth removing, and +leaving the files alone is the right answer. See [qc-report.md](qc-report.md). + +**What does MARS change in my file?** +The m/z arrays of MS2 spectra it corrected, and nothing else. Intensities, chromatograms and +metadata are untouched, and the bytes MARS did not mean to change are the input's own bytes. +See [mzml-passthrough.md](mzml-passthrough.md) and +[algorithm.md](algorithm.md#step-4-correction). + +**How much improvement should I expect?** +On Thermo Stellar ion-trap DIA, roughly half the median absolute fragment mass error. On an +already well-calibrated Astral run, under 2%. See [algorithm.md](algorithm.md#results). + +**Which library should I use?** +A Skyline PRISM report if you have one, because its `Product Mz` is genuinely theoretical +and it carries real per-peptide elution windows. A `.blib` without peak annotations will be +refused outright, and MARS will say so. See [spectral-libraries.md](spectral-libraries.md). + +**Is the output reproducible?** +Identical input gives a bit-identical model and bit-identical decoded output at any thread +count, enforced by its own CI job. The compressed bytes are not identical across platforms, +because runtimes ship different zlib builds - use `mars compare`, not `cmp`. See +[model.md](model.md#determinism). + +**Does it give the same answer as the Python version?** +Fragment matching and every model feature are bit-identical: 160,947 fragments across two +Stellar runs, 24 columns, maximum absolute difference zero. The models are different +implementations, agree to r = 0.9955, and leave the same amount of error behind. Four +Python defects are deliberately not reproduced. See [python-parity.md](python-parity.md) +and [model.md](model.md#how-close-is-this-to-xgboost). + +**Something looks wrong with a corrected file.** +Run `mars verify` on the *input*. It round-trips the file with a null correction and checks +the index, checksum and decoded arrays. If that fails, the problem is file handling rather +than the model, and nothing else is worth investigating first. + +--- + +For installing and running MARS, see the [top-level README](../README.md). For the C# +source tree, see [dotnet/README.md](../dotnet/README.md). diff --git a/docs/algorithm.md b/docs/algorithm.md new file mode 100644 index 0000000..28e3029 --- /dev/null +++ b/docs/algorithm.md @@ -0,0 +1,358 @@ +# The MARS recalibration algorithm + +How MARS turns a spectral library and a set of DIA runs into corrected m/z values. + +- [The problem](#the-problem) +- [Overview](#overview) +- [Step 1: Fragment matching](#step-1-fragment-matching) +- [Step 2: Feature extraction](#step-2-feature-extraction) +- [Step 3: Model training](#step-3-model-training) +- [Step 4: Correction](#step-4-correction) +- [What the model actually learns](#what-the-model-actually-learns) +- [Determinism](#determinism) +- [Limits](#limits) + +## The problem + +A mass analyzer's reported m/z drifts. On a Thermo Stellar ion trap the drift is large +enough to matter: across a five-file HeLa GPF-DIA cohort the fragment mass error has a +standard deviation of **0.118 Th** and a median absolute deviation of **0.080 Th**, with +a systematic offset of about -0.013 Th. + +That error is not random. It varies with where the peak sits in the scan, how much signal +the trap accumulated, how crowded the neighborhood is, and when in the run the scan +happened. Anything systematic is learnable, and anything learnable can be subtracted. + +MARS learns the systematic part from peaks whose true m/z is already known - fragments of +peptides a spectral library has identified - and applies the learned correction to every +peak in the file. + +## Overview + +``` + spectral library mzML (MS2) + | | + +--------------+-------------------+ + | + [1] fragment matching + most intense peak within tolerance + | + delta_mz = observed - theoretical + | + [2] feature extraction + 22 features per matched peak + | + [3] model training + gradient boosted trees, squared error + | + model.json + | + [4] correction + corrected = observed - predicted error + | + {input}-mars.mzML +``` + +MARS makes **two passes** over each file. The first reads spectra, matches fragments and +builds training rows. The second re-reads the file and splices corrected m/z arrays into a +byte-for-byte copy of the original. Neither pass holds the file in memory; a 4.9 GB Astral +run streams in the same working set as a 1.2 GB Stellar one. + +## Step 1: Fragment matching + +A library fragment is compared against a spectrum only when the spectrum could plausibly +contain it: + +1. **Isolation window.** The library precursor m/z must fall inside the spectrum's DIA + isolation window, `low <= precursor_mz <= high`, both bounds inclusive. +2. **Retention time.** The spectrum's scan start time must fall inside the library entry's + RT window, `rt_start <= rt <= rt_end`, both inclusive. Entries with no RT window are + considered at every retention time. + +Within a candidate spectrum, each fragment is looked for within a tolerance of its +theoretical m/z - either absolute (`--tolerance`, default 0.3 Th) or relative +(`--tolerance-ppm`, appropriate for Orbitrap and Astral data). + +> **MARS takes the MOST INTENSE peak in the window, not the nearest.** +> +> This is the single most important choice in the matching step. Taking the nearest peak +> would bias every label toward zero - you would be selecting for peaks that agree with +> the library and measuring the error you had already assumed. The most intense peak has +> the best-determined centroid, and whatever error it carries is the error worth learning. + +Peaks below `--min-intensity` (default 500) cannot become training rows. The threshold +governs training only; **every** peak in a corrected spectrum is corrected regardless of +intensity. + +The label is: + +``` +delta_mz = observed_mz - theoretical_mz +``` + +Positive means the instrument reported the peak too high. There is no q-value filter and +no outlier trimming: confidence comes from the library having been built from confident +identifications, and the tolerance bounds the label by construction. + +## Step 2: Feature extraction + +Each matched fragment becomes one row of up to 22 features. The order below is the model's +feature order and is part of the on-disk model contract. + +**Where the peak is, and how strong** + +| Feature | Meaning | +|---|---| +| `precursor_mz` | Isolation window target m/z. Different DIA windows can calibrate differently. | +| `fragment_mz` | The reference m/z. Mass error is mass-dependent, so this is the backbone of the model. | +| `log_intensity` | log10 of peak intensity. A weak peak has a noisier centroid. | +| `log_tic` | log10 of the spectrum's summed intensity. A proxy for total trap loading. | + +**How much charge was in the trap** + +| Feature | Meaning | +|---|---| +| `injection_time` | Ion injection time, in seconds. | +| `tic_injection_time` | Summed intensity x injection time: total ions rather than an ion rate. | +| `fragment_ions` | Peak intensity x injection time: this peak's ion count. | + +**What was next to the peak** (the space-charge features) + +Six windows of summed neighbor intensity, each 1 Th wide, scaled by injection time: + +| Feature | Window, relative to the reference m/z x | +|---|---| +| `ions_above_0_1` | (x + 0.5, x + 1.5] | +| `ions_above_1_2` | (x + 1.5, x + 2.5] | +| `ions_above_2_3` | (x + 2.5, x + 3.5] | +| `ions_below_0_1` | (x - 1.5, x - 0.5] | +| `ions_below_1_2` | (x - 2.5, x - 1.5] | +| `ions_below_2_3` | (x - 3.5, x - 2.5] | + +Plus six ratio features, `ions_* / fragment_ions`, expressing the neighborhood relative to +the peak itself rather than in absolute terms. + +The half-Th offsets are deliberate: isotopes sit at +1, +2, +3 Th, so a window running from ++0.5 to +1.5 brackets the first isotope instead of straddling two. Bounds are exclusive at +the low end and inclusive at the high end. A window that runs past the edge of the recorded +scan range simply finds nothing and sums to zero - there is no missing-value marker. + +**When, and how hot** + +| Feature | Meaning | +|---|---| +| `absolute_time` | Seconds since the earliest acquisition in the cohort. Captures drift across a run sequence, not just within one file. | +| `rfa2_temp`, `rfc2_temp` | RF generator temperatures at that retention time, when temperature logs are supplied. | + +### Features are selected, not assumed + +Not every run supports every feature, and two different things can be missing. + +**No injection time at all.** A file with no `MS:1000927 ion injection time` cvParam loses +fifteen of the twenty-two at once - `injection_time`, `tic_injection_time`, `fragment_ions`, +the six neighbor windows and the six ratios. All of them count ions, and without an injection +time there is nothing to turn an ion rate into an ion count with. Bruker and Sciex files +tested here are in this position. + +**An injection time that never moves.** Only `injection_time` and `tic_injection_time` go: a +constant cannot be split on, and TIC times a constant is `log_tic` rescaled. The other +thirteen stay. They are scaled by the injection time but are not it, and multiplying a column +by a positive constant does not reorder it, so every split a tree could have made on them is +still available. + +Whether it moves is decided from the whole matched column, not from a sample of the head of +the run. An ion trap holds its injection time at the method's ceiling until the trap actually +fills, which on a gradient is the entire void volume - so a run judged on its first few hundred +spectra reads as constant no matter what it does later. Every Stellar run tested does exactly +that, one of them with two thirds of its spectra off the ceiling. + +The fitted model records the names it actually used, and loading a model whose feature list +does not match the extractor is a hard error rather than a warning. + +**No feature is ever NaN.** Rows that cannot supply a selected feature are dropped before +training. This is a hard requirement, not tidiness: the tree implementation maps NaN to the +lowest bin while the tree walk sends it right, so a NaN reaching the model would be routed +inconsistently between fitting and scoring. + +## Step 3: Model training + +Gradient boosted trees with a squared-error objective, from +[`Osprey.ML`](https://github.com/ProteoWizard/pwiz/tree/master/pwiz_tools/Osprey/Osprey.ML) - +the same implementation Osprey uses for FDR scoring, so there is one boosting implementation +to maintain rather than one per tool. + +| Hyperparameter | Value | +|---|---| +| objective | squared error | +| boosting rounds | 100 | +| max depth | 6 | +| learning rate | 0.1 | +| min child weight | 1.0 | +| subsample / colsample | 1.0 / 1.0 | +| L2 (lambda) / L1 (alpha) | 1.0 / 0.0 | +| histogram bins | 256 (clamped to 255) | +| seed | 42 | + +Rows are weighted by observed peak intensity, normalized to mean 1. The normalization +matters: under squared error the hessian **is** the sample weight, so raw detector counts +would put the summed hessian in the millions and make `min_child_weight` meaningless. + +By default MARS trains five models, one per fold, with **folds split by peptide**, and +every reported number comes from a model that never saw the peptide it is scoring. A +peptide's fragments recur across hundreds of spectra with the same theoretical m/z, and +`fragment_mz` is a feature, so splitting rows rather than peptides would let the model +memorize a peptide's error and report an accuracy it cannot reach on anything new. See +[model.md](model.md#cross-validation). + +> **A note on `min_child_weight`.** It thresholds the summed hessian, and the hessian means +> different things under different objectives. Under logistic loss it is p(1-p), never above +> 0.25, so a threshold of 1.0 means several samples. Under squared error it is the weight, +> so with unit weights 1.0 means exactly **one** sample. Hyperparameters do not transfer +> between objectives. + +Below `--min-training-rows` (default 1000) MARS refuses to fit and exits 2, rather than +producing a model built on noise. + +[model.md](model.md) goes further: why boosted trees rather than a polynomial, how the +objective and histogram splits work, how the model file is laid out, and how the +predictions compare against Python's XGBoost on identical rows. + +## Step 4: Correction + +For every peak of every qualifying MS2 spectrum: + +``` +corrected_mz = observed_mz - model.PredictDelta(features) +``` + +The sign follows from the label: `delta_mz` is observed minus theoretical, so the predicted +error is subtracted. Getting this backwards roughly doubles the error instead of halving it, +which is at least a loud failure. + +Two differences from training are worth knowing: + +- **`fragment_mz` is the observed m/z**, because at correction time there is no library to + supply a theoretical one. The two differ by at most the matching tolerance. +- **The neighbor windows are anchored on each observed peak**, for the same reason. + +MS1 spectra, intensity arrays, chromatograms and all metadata are untouched. Spectra whose +isolation window is wider than `--max-isolation-window` are skipped entirely. + +### Keeping m/z ascending + +A per-peak correction can in principle reorder two adjacent peaks, and mzML consumers assume +a sorted m/z array. `--on-reorder` chooses what happens: + +| Mode | Behavior | +|---|---| +| `clamp` (default) | Raise the offending peak to the next representable double above its predecessor. | +| `revert` | Leave that whole spectrum uncorrected. | +| `allow` | Write the values as they came out. For diagnosing how often it happens. | + +Violations are counted and reported under every mode. On the reference Stellar cohort the +count is **zero** across 565,498 corrected spectra - corrections are two orders of magnitude +smaller than typical peak spacing, so this is a guard against a pathological model rather +than a routine event. + +## What the model actually learns + +Feature importance from the reference Stellar cohort, as reported by the Python +implementation's gain importance: + +| Feature | Importance | +|---|---| +| `ions_above_0_1` | 0.346 | +| `adjacent_ratio_0_1` | 0.292 | +| `fragment_mz` | 0.156 | +| everything else | < 0.02 each | + +Two thirds of the model is the **first neighbor window above the peak** and its ratio to the +peak's own intensity. That is a space-charge effect: an ion cloud sitting roughly one Th +above a peak perturbs the measured m/z of that peak, and the size of the perturbation scales +with how much charge is there relative to the peak itself. The mass-dependence term +(`fragment_mz`) comes third. + +This is why the six neighbor windows exist at all, and why the injection-time scaling +matters: what perturbs the measurement is the number of ions, not the rate at which they +arrived. + +### Results + +Measured on the corrected files, by rematching the library against the written output: + +| Stellar HeLa GPF-DIA, 5 files | Uncorrected | Corrected | +|---|---|---| +| Fragments matched | 352,349 | 358,334 | +| Mean delta m/z | -0.0134 Th | -0.0023 Th | +| Median delta m/z | -0.0082 Th | -0.0025 Th | +| Std delta m/z | 0.1180 Th | 0.0872 Th | +| MAD delta m/z | 0.0800 Th | 0.0464 Th | +| RMS delta m/z | 0.1188 Th | 0.0872 Th | + +A 42% reduction in MAD and a 26% reduction in standard deviation. + +Twenty features are used on this cohort. Most of the gain comes from the thirteen +ion-population features - `ions_above_0_1` alone carries the highest permutation importance of +any feature in the model. See the note on them and on injection time under +[the CLI reference](cli-reference.md); the release notes record what switching them off costs, +which is most of the correction. + +**MARS does not help every instrument.** On an Astral plate the same pipeline moves the +spread by under 2%, because the data arrives already calibrated to about 4 ppm and there is +essentially nothing systematic left to remove. That is a real result about the data, not a +failure of the method: run `mars qc` first and see whether there is anything to correct +before correcting it. + +## Determinism + +MARS writes m/z values into files that get reprocessed and compared, so determinism is a +correctness requirement rather than a nicety. + +**Identical input produces identical decoded m/z values** - on any thread count, on any +platform, on every run. The guarantees behind that: + +- Histogram accumulation parallelizes **across features only**. One thread owns one + feature's histogram and walks the node's rows in ascending order, so no summation order + can depend on the thread count. +- Subsampling draws from a seeded `XorShift64`, never `System.Random`, whose seeded stream + is a runtime implementation detail. +- Split selection walks features and bins in ascending order and takes a new best only on a + strict improvement, so ties resolve to the lowest (feature, bin). +- Row partitioning is stable. +- Inference has no cross-row accumulation, so parallelizing it cannot change a value. + +**File bytes are a different matter.** Compressed bytes are *not* guaranteed identical +across platforms, because the zlib each runtime ships is not the same. Verified on the same +input: + +| | Windows | Linux | +|---|---|---| +| Output size | 1,176,380 bytes | 1,176,172 bytes | +| Raw bytes | differ | | +| Decoded m/z and intensity | **identical**, 0 of 10,283 peaks differ | | + +Equivalence is defined on decoded values, which is what any consumer actually reads. Use +`mars compare a.mzML b.mzML` to check two files on that basis rather than with `cmp`. + +## Limits + +- **Centroided input is assumed.** Neither implementation currently detects or rejects + profile-mode spectra. +- **MS2 only.** MS1 peaks are never corrected. +- **One model per invocation**, fitted across all input files together. That is what makes + `absolute_time` meaningful - it spans the cohort, so the model can learn drift across a + run sequence. +- **The correction is only as good as the library.** A library whose fragment m/z values + are observed rather than theoretical teaches the model to reproduce another run's + calibration error. See [spectral-libraries.md](spectral-libraries.md). + +## See also + +- [model.md](model.md) - the gradient boosted trees in depth, and the model file format +- [spectral-libraries.md](spectral-libraries.md) - the four library sources and their quirks +- [qc-report.md](qc-report.md) - how to read the figures MARS writes +- [mzml-passthrough.md](mzml-passthrough.md) - how the output file is written +- [cli-reference.md](cli-reference.md) - every command and option +- [architecture.md](architecture.md) - a map of the code +- [python-parity.md](python-parity.md) - how this is checked against the Python implementation +- [dotnet-port-spec.md](dotnet-port-spec.md) - the port specification and acceptance gates diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..6527f38 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,224 @@ +# How MARS is put together + +A map of the code, for anyone modifying it. For what the tool does, start with +[the algorithm](algorithm.md). + +## Projects + +``` +dotnet/ + MARS/ the CLI: argument parsing, the five commands, the QC report + MARS.Core/ matching, features, the calibrator, correction. No I/O + MARS.IO/ mzML reading and writing, library readers, the SQLite reader + MARS.Pwiz/ vendor formats in, non-mzML formats out, through pwiz-sharp. Optional + MARS.OspreyML/ compiles the vendored Osprey.ML sources + MARS.Test/ 194 tests, 182 of them without a pwiz checkout + third_party/ + Osprey.ML/ verbatim copies of the boosting code, hash-guarded +``` + +The split that matters is `MARS.Core` having no I/O. Matching, feature extraction and +correction take arrays and return arrays, which is what makes them testable without +fabricating files and what keeps the parallel correction path free of anything that could +touch a stream. + +`MARS.Core` is also the only assembly with no package references at all. `MARS.IO` has one, +`Parquet.Net`, and that is the only native code in the tree. See +[dependencies](#dependencies). + +## What happens during `mars calibrate` + +``` + library file mzML files + | | + PrismCsv / Blib / DiannParquet MzMLFile.Inspect + LibraryReader (index, byte offsets) + | | + SpectralLibrary <---- FragmentMatcher ----> SpectrumRecord stream + (column arrays) | + v + MatchTable one row per matched fragment, + (column-major) 22 feature columns + | + v + MzCalibrator.Fit + | + +--------------+--------------+ + | | + MzCalibrator QcHtmlReport + | QcReport + v + SpectrumCorrector ----> MzMLWriter (byte splice) + | + v + {input}-mars.mzML +``` + +Two passes over each file: one to match, one to correct. The alternative - hold every +spectrum in memory and correct in place - is what makes a naive implementation fail on a +5 GB Astral run. + +## Data structures worth knowing + +**`SpectralLibrary`** is column-major, not a list of objects. `FragmentStart[entry]` indexes +into flat `FragmentMz` / `FragmentIonType` / `FragmentCharge` arrays. A plate-scale Skyline +report is 67 million rows; a managed object per fragment would not fit. + +**`MatchTable`** is likewise column-major, one `GrowableArray` per feature. It is +also exactly the layout the model wants, so training does not copy. Detail columns (scan +number, fragment index, observed m/z, retention time) are allocated only when something +needs them - a dump or the HTML report - because they cost 16 bytes a row across millions +of rows. + +**`SpectrumRecord`** carries the decoded m/z and intensity arrays plus the metadata the +features need. Buffers are pooled and reused between spectra. + +## The pwiz path + +`MARS.Pwiz` is where every format that is not mzML enters or leaves. It is **optional**: +pwiz-sharp has no package feed, so the reference points at an external checkout and the project +compiles either way. Without one, `MARS_NO_PWIZ` drops the backend, `PwizOutput` reports itself +unavailable, and MARS reads and writes mzML exactly as it always has. That is the same shape +pwiz's own vendor projects use for their SDKs. + +| Type | Does | +|---|---| +| `SpectrumSources` | Picks a reader from the path: mzML to `MARS.IO`, everything else to pwiz | +| `PwizSpectrumSource` | Vendor formats in, as `SpectrumRecord` - the same type the mzML reader yields | +| `MarsSpectrumList` | A pwiz `SpectrumList` that applies the correction as spectra are pulled through | +| `PwizWriteBackend` | mzXML, mzMLb and mgf out, and mzML when the input was a vendor file | +| `VendorReaders` | Registers the vendor readers with pwiz's dispatcher, from a module initializer | +| `MzMLEncoding` | Reads the input's binary encoding so the output matches it | + +Three things about it are load-bearing: + +**`ISpectrumSource` is the seam**, and it lives in `MARS.Core`. The matcher, the features and +the model consume `SpectrumRecord` and neither know nor care which reader produced it, so +`MARS.Core` depends on neither `MARS.IO` nor `MARS.Pwiz`. + +**The correction is applied identically on both paths.** `MarsSpectrumList` calls the same +`SpectrumCorrector` the byte-splice writer does. Writing one file both ways and diffing with +`mars compare` found no difference across 82,349,582 peaks. + +**Ion mobility is collapsed, not modelled.** pwiz is asked to combine each TIMS frame's +mobility scans into one spectrum per isolation window, on read and on write. Uncombined, a +diaPASEF frame is hundreds of two-peak spectra sharing one retention time, and twelve of +MARS's features are computed from the peaks surrounding a match. + +## The mzML path + +The one design decision that shapes everything else: **MARS does not parse mzML into a +document and write it back out.** It scans for spectrum elements, decodes only the binary +arrays it needs, and splices corrected arrays into a byte-for-byte copy of the input. + +- `MzMLSpanScanner` walks the file finding element boundaries without building a tree. +- `MzMLSpectrumParser` pulls out the cvParams and binary arrays for one spectrum. +- `MzMLBinaryCodec` does base64 and zlib. +- `MzMLWriter` copies input bytes through, substituting re-encoded m/z arrays and fixing up + the index offsets and checksum. + +Consequences, good and bad, are in [mzML passthrough](mzml-passthrough.md). The short +version: an entire class of "MARS broke my file" problems cannot happen, because the bytes +MARS did not mean to change are the input's bytes. + +## Library readers + +Three formats, one output type: + +| Reader | Source | Notes | +|---|---|---| +| `PrismCsvLibraryReader` | Skyline transition report | Streams; the reference report is 16.1 GB | +| `BlibLibraryReader` | BiblioSpec `.blib` | Via the managed SQLite reader | +| `DiannParquetLibraryReader` | DIA-NN `report-lib.parquet` | Needs `report.parquet` for RT | + +They differ in more than format - they differ in whether their m/z values are theoretical at +all, which is the single most important thing about a MARS library. See +[spectral libraries](spectral-libraries.md). + +## The managed SQLite reader + +`.blib` is a SQLite database, and MARS reads it with a small reader written for this purpose +rather than `Microsoft.Data.Sqlite`. + +The reason is dependency shape. `Microsoft.Data.Sqlite` brings `SQLitePCLRaw` and a native +`e_sqlite3` for every runtime identifier - exactly the per-platform native payload the port +exists to avoid. A BiblioSpec library only ever needs sequential scans of a handful of +tables, which is a small and well-specified subset of the format. + +What it supports: + +- Table b-trees, interior and leaf pages +- Overflow page chains, for rows too large for one page +- The record format: varints, serial types, and the type-code encoding of integers, floats, + text and blobs +- UTF-8 and UTF-16 text + +What it does not: indices, WAL, encryption, writing. None of which a library scan needs. + +**The one subtlety that caused a real bug.** SQLite's `INTEGER PRIMARY KEY` *aliases the +rowid*: the column is stored as NULL in the record, and the value lives in the b-tree's +rowid. A reader that trusts the record gets NULL for every id. This surfaced as reading 1 +precursor out of 8,587 - the join silently matched nothing - which is the characteristic +failure of a hand-written binary format reader: not a crash, just quietly wrong data. The +reader now falls back to `row.RowId` when such a column is null. + +That episode is also why this reader is on the list of things most deserving of more tests; +see [the coverage note](#testing). + +## Determinism + +Identical input, bit-identical output, at any thread count. Enforced by a dedicated CI job. + +- Model training parallelizes across features only, never across rows, so no float + accumulation is split across threads. +- The correction pass parallelizes across spectra, and each spectrum's output bytes are + independent of the others. +- Seeded XorShift64 for subsampling and tie-breaking. + +The exception is compressed bytes: different platforms ship different zlib builds, so output +files are not byte-identical across platforms even though every decoded value is. Use +`mars compare`, not `cmp`. + +## Dependencies + +`MARS.Core` has none. `MARS.Pwiz` has whatever a pwiz-sharp checkout brings, and nothing when +there is not one. `MARS.IO` has `Parquet.Net`, which brings `IronCompress` and a native +compression library - the only native code in the tree. + +It is confined to the DIA-NN path, and it does not fail closed: without the native library, +Snappy (what DIA-NN writes), Gzip, Brotli, Zstd and uncompressed all still work through +managed fallbacks, and only LZ4 and LZO fail, with a message naming the codec. That matters +on the two shipped platforms `IronCompress` publishes no native for: **Windows on Arm and +Intel macOS**. + +Splitting `DiannParquetLibraryReader` into its own assembly would restore the pure-managed +property for consumers that do not read DIA-NN libraries. Not done yet. + +## Testing + +194 tests. The parts with the strongest evidence are not the ones with the most tests: + +- **Fragment matching and every model feature** are verified against the Python + implementation row by row - 160,947 fragments, 24 columns, maximum absolute difference + zero. See [parity](python-parity.md). This is stronger evidence than unit tests with + invented expected values. +- **mzML passthrough** is covered by round-trip tests and by `mars verify` on real files. +- **The vendored boosting code** is hash-guarded against upstream and bit-identity-checked. + +The thinnest areas, in rough order of risk: the managed SQLite reader and the `.blib` path +(exercised once, manually, and it was wrong the first time); `PrismCsvLibraryReader`'s +replicate filtering and de-duplication; and `CommandLineArgs`, where a parsing bug means +silently running with the wrong tolerance. + +## Where things live + +| I want to change... | Look at | +|---|---| +| which peak gets matched | `MARS.Core/FragmentMatcher.cs`, `PeakSearch.cs` | +| a feature's definition | `MARS.Core/MarsFeature.cs`, `FragmentMatcher.cs` | +| training or hyperparameters | `MARS.Core/MzCalibrator.cs` | +| the boosting itself | upstream in pwiz, then re-sync. Not the vendored copy | +| how corrected files are written | `MARS.IO/MzMLWriter.cs` | +| a library format | `MARS.IO/*LibraryReader.cs` | +| the QC figures | `MARS/Report/` | +| a command-line option | `MARS/*Command.cs` and its `PrintHelp` | diff --git a/docs/cli-reference.md b/docs/cli-reference.md new file mode 100644 index 0000000..7bcd956 --- /dev/null +++ b/docs/cli-reference.md @@ -0,0 +1,551 @@ +# Command-line reference + +Five commands. `calibrate` is the one that does the work; the other four exist to tell you +whether you should trust it. + +| Command | What it does | Writes mzML | +|---|---|---| +| [`qc`](#mars-qc) | Reports the mass accuracy already in the files | no | +| [`calibrate`](#mars-calibrate) | Learns a correction and applies it | yes | +| [`apply`](#mars-apply) | Reuses a trained model on more files | yes | +| [`verify`](#mars-verify) | Round-trips a file and checks it survived | yes, then deletes it | +| [`compare`](#mars-compare) | Diffs two mzML files on decoded values | no | + +A sensible first session: + +```bash +mars qc --mzml-dir runs/ --prism-csv report.csv # is there anything to correct? +mars verify runs/one.mzML # can MARS handle this file at all? +mars calibrate --mzml-dir runs/ --prism-csv report.csv --output-dir corrected/ +``` + +## Exit codes + +Worth wiring into a pipeline, because they distinguish "your data was not suitable" from +"MARS is broken". + +| Code | Meaning | +|---|---| +| 0 | Success | +| 1 | Input error: files not found, a required option missing, an unreadable library | +| 2 | Not enough training data. Fewer matches than `--min-training-rows`; no model was fitted and nothing was written | +| 3 | Output validation failed. A file was written but did not pass its own index/checksum check - treat the output as suspect and report it | + +Exit 2 is a deliberate refusal rather than a crash. MARS would rather write nothing than +fit a model on a few hundred noisy rows and hand back a file that looks corrected. + +--- + +## Resolution and tolerance + +MARS reads the mass analyzer out of the mzML and picks a fragment tolerance to match, so +neither `qc` nor `calibrate` normally needs to be told what instrument produced the data. + +| `--resolution` | Meaning | +|---|---| +| `auto` (default) | Read the analyzer from the file's `instrumentConfiguration` and pick | +| `unit` | Ion trap or quadrupole: 0.3 Th, QC report in Th | +| `hram` | Orbitrap, FT-ICR, TOF or Astral: 10 ppm, QC report in ppm | + +The mode sets **defaults only**. An explicit `--tolerance` or `--tolerance-ppm` always wins; +detection can be wrong on a file MARS has not seen the shape of, and the person running it +can be certain in a way a heuristic cannot. + +Detection reads the analyzer for the **MS2** spectra specifically, which on a hybrid +instrument is not the one the run names as its default. An Orbitrap Astral file declares two +configurations - the orbitrap that takes the MS1 survey, which is the run default, and the +Astral analyzer that only the MS2 spectra point at. MS2 is what MARS calibrates, so that is +the one that decides. + +The choice appears in the log: + +``` +INFO high-resolution data; fragment tolerance 10 ppm (--tolerance or --tolerance-ppm to override) +INFO unit-resolution data; fragment tolerance 0.3 Th (--tolerance or --tolerance-ppm to override) +``` + +When the file does not say, MARS warns and falls back to 0.3 Th rather than guessing +silently: + +``` +WARNING: could not tell the mass analyzer from the file; assuming unit resolution and a +0.300 Th tolerance. Pass --resolution hram or --tolerance-ppm if this is Orbitrap, TOF or +Astral data. +``` + +One tolerance is chosen for the whole cohort, from the first file. If another file in the run +was recorded on a different kind of analyzer, MARS says so - a directory holding both trap and +high-resolution data gets one of them matched at the wrong width otherwise: + +``` +WARNING: astral.mzML was recorded on a high-resolution analyzer, but the fragment tolerance is +being set from a unit-resolution one. One tolerance is used for the whole cohort; calibrate the +instruments separately, or set --resolution to choose deliberately. +``` + +It is a warning rather than a refusal, because a mixed cohort can be deliberate and +`--resolution` is there to settle it. A file that does not name its analyzer is not treated as +a disagreement: it says nothing, and it already falls back to the default. + +Getting this wrong is quiet rather than loud, which is why it is detected. See +[choosing a tolerance](spectral-libraries.md#choosing-a-tolerance) for what a mismatched +window actually does to the numbers. + +--- + +## Input formats + +MARS reads mzML itself. With a pwiz-sharp build it also reads Thermo `.raw` directly, so a +run can be calibrated straight off the instrument with no conversion step: + +```bash +mars calibrate --mzml run.raw --library report-lib.parquet --diann-report report.parquet --output-dir corrected/ --output-format mzMLb +``` + +`--mzml`, `--mzml-dir` and bare file arguments all accept any readable format; the name is +historical. A directory picks up every file MARS can read, not only `.mzML`. + +Reading a `.raw` gives the same answer as reading the mzML msconvert would have made from it. +On an Astral run matched against the same DIA-NN library, both paths return 230,781 fragment +matches with the same median, standard deviation and MAD to every reported digit. + +It is not faster to *read* - that run takes 53 s from `.raw` against 15 s from the converted +mzML, and vendor reading does not thread - so the saving is the conversion that no longer has +to happen and the intermediate file it no longer leaves behind. + +The mass analyzer is detected from the vendor file exactly as it is from an mzML, so an Astral +`.raw` picks a 10 ppm tolerance and a ppm-scaled QC report on its own. + +### Platforms + +All six release targets build and run with the vendor reader, because the Thermo SDK MARS uses +is managed and cross-platform. + +| Target | Reads `.raw` | mzML, mzXML | mzMLb | +|---|---|---|---| +| `win-x64`, `linux-x64`, `osx-x64` | yes | yes | yes | +| `win-arm64`, `linux-arm64`, `osx-arm64` | yes | yes | **no** | + +mzMLb is the one gap: it is HDF5, and `HDF.PInvoke.1.10` bundles a native libhdf5 for x64 only, +so an arm64 build has nothing to write it with. Everything else - reading vendor files +included - works on all six. + +A build also stages two files it never uses on non-Windows targets: `MassLynxRaw.dll`, which is +a Windows native library, and a Waters `license.key`. Both arrive through the same transitive +Thermo-to-Analysis-to-Waters reference that costs the IL3000 suppression, and both are inert. + +### Which vendors + +| Format | Vendor | Status | +|---|---|---| +| `.mzML` | - | Always, read by MARS itself | +| `.raw` | Thermo | Windows, Linux, macOS | +| `.d`, `.tdf`, `.tsf`, `.baf` | Bruker | Windows and Linux | +| `.wiff`, `.wiff2` | Sciex | **Windows only** | +| `.lcd` | Shimadzu | Not referenced yet | +| `.d` | Agilent | Not referenced yet | + +Sciex is Windows-only because its SDK is: a SmartAssembly bundle loaded through a side-by-side +`AssemblyLoadContext`, needing a native SQLite interop, which pwiz-sharp gates on Windows for +that reason. Thermo's SDK is managed and runs anywhere. Bruker ships separate Windows and +Linux archives. + +Bruker and Agilent runs are **directories** rather than files. `--mzml`, `--mzml-dir` and bare +arguments all accept them, and pwiz decides which vendor a `.d` belongs to by what is inside +it. + +Verified against ProteoWizard's own vendor test files: a Bruker `diaPASEF.d`, a Sciex ZenoTOF +7600 `.wiff2`, a Sciex SWATH `.wiff2`, and a legacy `.wiff`. All were detected as +high-resolution except the legacy `.wiff`, which is unit-resolution, and all reported isolation +windows on every MS2. + +**Injection time has to vary to be a feature.** A trap sets it per spectrum from its automatic +gain control, so it says how full the trap was. An instrument that accumulates for a fixed +period gives the same number every time, and then `injection_time` is a constant - which a tree +can never split on - while `tic_injection_time` is TIC times that constant, which is `log_tic` +rescaled. Two features carrying nothing, one of them a duplicate that splits permutation +importance with the feature it duplicates. + +**That is decided from the whole run, not a sample of it.** A trap sits at the method's ceiling +until the trap actually fills, so the start of a gradient is flat no matter what follows - +every Stellar run tested is constant over its first several thousand MS2 and varies later, one +of them across two thirds of its spectra. MARS therefore collects the column during matching +and decides afterwards, when it has all of it. Nothing extra is read to do this. + +**The ion-population features are a different question.** `fragment_ions`, the six +`ions_above_`/`ions_below_` windows and the six `adjacent_ratio_` features are peak sums over +m/z windows, multiplied by the injection time to turn a rate into a count. They need an +injection time to exist, but not to vary: a constant multiplies every one of them by the same +factor, which leaves them all varying and every split still available. So they are kept +whenever the run records an injection time at all, and dropped only when it records none. + +Both cases are reported: + +``` +WARN No ion injection time in this run; the ion-population features are off. +INFO ion injection time is the same on every matched spectrum; injection_time and + tic_injection_time are off. The ion-population features stay. +``` + +On the Bruker and Sciex files tested here the cvParam is absent altogether rather than +constant, so the first message applies; the second covers an instrument that genuinely +accumulates for a fixed period. + +---|---|---| +| `.mzML` | - | Always | +| `.raw` | Thermo | With a pwiz-sharp build | +| `.wiff`, `.wiff2` | Sciex | Not referenced yet; Windows-only when it is | +| `.d` | Agilent, Bruker | Not referenced yet | +| `.lcd` | Shimadzu | Not referenced yet | + +--- + +## Output formats + +`calibrate` and `apply` write mzML by default. `--output-format` selects another. + +| `--output-format` | Written by | Notes | +|---|---|---| +| `mzML` (default) | MARS, or pwiz | Spliced when the input is mzML; built when it is a vendor file | +| `mzXML` | pwiz | Cannot express ion mobility or some isolation-window terms | +| `mzMLb` | pwiz | mzML in an HDF5 container; roughly half the size | +| `mgf` | pwiz | MS2 peak lists only - no MS1, no chromatograms, no scan metadata | + +Two different writers sit behind this, and which one runs depends only on the format. + +**mzML is spliced when there is something to splice.** MARS copies the input and replaces +only the m/z arrays it corrected, so everything else is identical by construction rather than +by care. That is the whole of [the passthrough contract](mzml-passthrough.md), and it is why +mzML is the default. + +Splicing needs an mzML to copy. Reading a `.raw` and writing mzML has nothing to splice into, +so that file is built by pwiz like any other format - the guarantee applies to mzML in and +mzML out, and is not pretended at otherwise. + +**Everything else is built.** There is no input of that format to splice into, so the file is +serialized from scratch by [pwiz-sharp](https://github.com/ProteoWizard/pwiz/pull/4178) - the +same code msconvert uses, which is also what wrote the mzML MARS is reading. Both paths run +the same correction over the same values: writing one file both ways and diffing them with +`mars compare` finds no difference across 82 million peaks. + +The binary encoding is read from the input and matched, per array, so a 64-bit zlib input +produces a 64-bit zlib output. Left to its own defaults pwiz writes 64-bit **uncompressed**, +which inflated a Stellar run by 61%. + +`mgf` and `mzXML` print a warning at startup saying what they drop. + +### Threads + +`--threads` defaults to `auto`, which is one worker per logical processor. The run says which +it chose: + +``` +INFO Using 16 worker threads, one per logical processor. --threads to change it. +``` + +One number drives all three parallel stages - the mzML writer, the pwiz spectrum list, and the +histogram build inside the boosting implementation. Matching is not one of them: it streams +spectra in order on a single thread, so a full `calibrate` never speeds up in proportion to +this. Nothing about the correction depends on the count; it is a CPU-use knob, not an accuracy +one, and the output is identical at any setting. + +Whether the hardware threads of a simultaneously-multithreaded CPU are worth using is usually +argued rather than measured, so it was measured. Correcting and rewriting one 1.2 GB Stellar +run on an 8-core i9-9900K with 16 logical processors, best of two passes taken in both +directions to spread thermal drift: + +| threads | 2 | 4 | 6 | 8 | 10 | 12 | 16 | +|---|---|---|---|---|---|---|---| +| seconds | 150.5 | 77.4 | 52.5 | 45.4 | 42.8 | 38.7 | 36.8 | + +Scaling is near-perfect to 4 and still improving at the end: **the 16 logical processors are +24% faster than the 8 physical ones**, so the default uses all of them and capping at physical +cores would give up most of a quarter of the throughput. + +It is a shallow curve past 8, though - about half the ideal speedup by 16 - and the writer +emits its finished spectra in order on one thread, which has to become the limit somewhere. +Where that falls on a 64- or 128-core machine has not been measured, so MARS imposes no +ceiling: a guessed one would be worse than none. It reports what it chose instead. + +`--threads` above the processor count is allowed and warned about; below 1 is refused, because +`--threads $N` with `N` unset should report a scripting mistake rather than quietly take the +whole machine. + +### Speed + +`--threads` applies to the pwiz writer as well as to MARS's own. Scoring the model is where a +conversion's time goes - on one Astral run, 243 s of 308 s, against 17 s reading and about +49 s encoding - and pwiz's writers pull spectra one at a time, so MARS reads a batch ahead and +corrects the batch in parallel. That run goes from 318 s on one thread to 103 s on twelve. + +Reads stay sequential: they are 5% of the work and the vendor readers are not thread-safe. +What is left after the model is parallelized is mostly pwiz's encoder, which is inside pwiz. + +### Byte-reproducibility + +mzML and mzXML are byte-reproducible: the same input, model and version produce the same +bytes, on any number of threads. Verified by writing an mzXML on 1 thread and on 12 and +comparing hashes. + +**mzMLb is not**, and not because of anything MARS does - two mzMLb writes of identical data, +at the same thread count, differ byte-wise, because the HDF5 container records things that +vary between writes. The spectra are the same; the file is not. Use mzML or mzXML where a +checksum has to match. + +### What can this binary do? + +`mars --version` reports what the binary in front of you actually carries, which is not a +property of MARS but of how it was built and where it is running: + +``` +26.1.0 +reads: .mzML, .raw, .wiff, .wiff2, .d, .tdf, .tsf, .baf +writes: mzML, mzXML, mzMLb, mgf +``` + +A build made without pwiz-sharp says `reads: .mzML` and `writes: mzML`. An arm64 build drops +Bruker, Sciex and mzMLb, because those need native x64 libraries while Thermo's SDK is managed. +The list is what the binary can do here, not what it recognizes the name of - a `.lcd` is +understood well enough to be refused with a reason, and is deliberately not advertised. + +### Builds without pwiz + +The pwiz reference is optional, because pwiz-sharp has no package feed yet. A MARS built +without it writes mzML and refuses the others with an explanatory error; nothing else about +MARS changes. To enable them, point the build at a pwiz checkout: + +```bash +dotnet build -c Release -p:PwizSharpDir=/path/to/pwiz/pwiz-sharp +``` + +`mars apply --validate` checks an mzML index and its SHA-1 footer. The other formats have +neither, so it says it is skipping rather than reporting a pass it did not make. + +--- + +## `mars qc` + +Matches library fragments against the spectra and reports the mass error that is already +there. Trains nothing, writes no mzML. + +**Run this first.** It answers the only question that matters before calibrating: is there +a systematic error here worth removing? On an already well-calibrated instrument the answer +is often no, and the honest outcome is to leave the files alone. + +```bash +mars qc --mzml-dir runs/ --prism-csv report.csv +mars qc --mzml-dir runs/ --library lib.blib --by-file +``` + +| Option | Meaning | +|---|---| +| `--mzml ` | mzML file or glob. Repeatable | +| `--mzml-dir ` | Directory of mzML files | +| `--prism-csv ` | Skyline PRISM report | +| `--library ` | `.blib`, DIA-NN `report-lib.parquet`, or a PRISM `.csv` | +| `--diann-report ` | DIA-NN `report.parquet`, for per-run RT windows | +| `--temperature-dir ` | Directory of `RFA2-`/`RFC2-` temperature CSVs | +| `--resolution ` | `unit`, `hram` or `auto` (default `auto`) | +| `--tolerance ` | Fragment tolerance in Th (default 0.3, or from `--resolution`) | +| `--tolerance-ppm ` | Fragment tolerance in ppm; overrides `--tolerance` | +| `--min-intensity ` | Minimum peak intensity to match (default 500) | +| `--max-isolation-window ` | Skip spectra with wider isolation windows | +| `--output ` | Text report path (default `mars_qc_summary.txt`) | +| `--html-report ` | Figures (default `mars_qc_report.html`, beside the text report) | +| `--no-html-report` | Skip the figures | +| `--by-file` | Report each input file separately rather than pooled | + +`qc` writes the same figures as `calibrate`, minus the ones that need a model: no corrected +distribution, no after-heatmap, no feature importance. What is left is the error as +measured and how it varies with each feature, which is exactly what the decision to +calibrate turns on. See [qc-report.md](qc-report.md). + +With `--no-html-report` only two features are collected, which is all the numbers need. +With figures on, every feature is collected so the panels mean something; the cost is one +pass over peaks MARS has already decoded. + +The text report gives the error in both Th and ppm, which is the quickest way to tell +whether your tolerance is sane before training anything. See +[choosing a tolerance](spectral-libraries.md#choosing-a-tolerance). + +--- + +## `mars calibrate` + +Matches, trains, and writes recalibrated files named `{input}-mars.mzML`. + +```bash +mars calibrate --mzml-dir runs/ --prism-csv report.csv --output-dir corrected/ +``` + +### Input + +| Option | Meaning | +|---|---| +| `--mzml ` | mzML file or glob. Repeatable | +| `--mzml-dir ` | Directory of mzML files | +| `--prism-csv ` | Skyline PRISM report CSV (theoretical `Product Mz`). Recommended | +| `--library ` | `.blib`, DIA-NN `report-lib.parquet`, or a PRISM `.csv` | +| `--diann-report ` | DIA-NN `report.parquet`, for per-run RT windows | +| `--temperature-dir ` | Directory of `RFA2-`/`RFC2-` temperature CSVs | + +Files can also be passed as bare arguments. All the input files are matched and trained on +together, which is the point: one model over the whole cohort sees more of the error +surface than one model per file. + +### Matching + +| Option | Default | Meaning | +|---|---|---| +| `--resolution ` | `auto` | `unit`, `hram` or `auto` | +| `--tolerance ` | 0.3 | Fragment tolerance in Th | +| `--tolerance-ppm ` | - | Fragment tolerance in ppm; overrides `--tolerance` | +| `--min-intensity ` | 500 | Minimum peak intensity to match | +| `--max-isolation-window ` | - | Skip spectra with wider isolation windows | +| `--rt-window ` | 0.083 | RT half-window around a `.blib` entry's library RT | +| `--no-dedupe-library` | off | Keep transitions that repeat across replicates | + +### Model + +| Option | Default | Meaning | +|---|---|---| +| `--n-estimators ` | 100 | Boosting rounds | +| `--max-depth ` | 6 | Tree depth | +| `--learning-rate ` | 0.1 | Shrinkage | +| `--robust ` | `trim` | Second pass over rows the first could not explain: `trim`, `huber`, or `none` | +| `--robust-sigma ` | 3 | Residual threshold for `--robust`, in robust sigma. 0 disables the second pass | +| `--cv-folds ` | 5 | Cross-validation folds, split by peptide. 0 trains a single model | +| `--validation-split ` | 0.2 | Held-out fraction, used only when `--cv-folds 0` | +| `--max-training-rows ` | no cap | Cap training rows by even stride | +| `--min-training-rows ` | 1000 | Refuse to fit below this many matches (exit 2) | +| `--seed ` | 42 | Random seed | + +The defaults are XGBoost's defaults, which is not a coincidence: see +[the model](model.md#hyperparameters). There is rarely a reason to change them. + +**Cross-validation does not change what gets applied.** The correction model is an +ordinary fit over every row - calibration is in-sample by nature - and the folds are a +measurement taken alongside it, answering what the same procedure would achieve on a run it +was not fitted to. So the cost is a few extra training rounds and nothing at correction +time: 52 s for `--cv-folds 0` against 66 s for the default, on one 1.47 GB Stellar file. +The report gives both numbers, labelled. See +[the model](model.md#calibration-is-in-sample-and-that-is-not-a-problem). + +### Output + +| Option | Meaning | +|---|---| +| `--output-dir ` | Output directory (default `.`) | +| `--model-path ` | Where to save the model (default `mars_model.json`) | +| `--report ` | Text QC summary (default `mars_qc_summary.txt`) | +| `--html-report ` | QC figures (default `mars_qc_report.html`) | +| `--no-html-report` | Skip the figures | +| `--dump-matches ` | Every matched fragment as CSV, with all computed features | +| `--dump-predictions ` | As `--dump-matches`, plus the model's prediction and residual | +| `--no-recalibrate` | Train and report only; write no mzML | +| `--on-reorder ` | `clamp` (default), `revert`, or `allow` | +| `--python-compat` | Reproduce two known Python inconsistencies, for A/B comparison | +| `--threads ` | Worker threads (default: auto, one per logical processor) | +| `-v, --verbose` | Verbose output | + +`--on-reorder` decides what happens when a per-peak correction would put an m/z array out +of ascending order. `clamp` nudges the offending value to preserve order, `revert` leaves +that whole spectrum uncorrected, `allow` writes it anyway. Violations are counted and +reported under every mode. See +[keeping m/z ascending](algorithm.md#keeping-mz-ascending). + +`--dump-matches` and `--dump-predictions` are diagnostics, and also the input to the +[parity harness](python-parity.md). A plate-scale cohort produces millions of rows. + +--- + +## `mars apply` + +Applies a model trained earlier to more files, with no rematching and no retraining. + +```bash +mars apply --model corrected/mars_model.json --mzml-dir more-runs/ --output-dir corrected/ +``` + +| Option | Meaning | +|---|---| +| `--model ` | Trained model. Required | +| `--mzml `, `--mzml-dir ` | Input files | +| `--output-dir ` | Output directory (default `.`) | +| `--temperature-dir ` | Temperature CSVs, if the model uses them | +| `--max-isolation-window ` | Leave wider isolation windows uncorrected | +| `--on-reorder ` | `clamp` (default), `revert`, or `allow` | +| `--python-compat` | Reproduce the Python inconsistencies | +| `--threads ` | Worker threads (default: auto, one per logical processor) | +| `--validate` | Check the index and checksum of each output | + +Use this for files acquired under the same conditions as the training set. A model carries +an `absolute_time` offset, and the feature list it was trained on; loading a model whose +features do not match what the extractor produces is a hard error rather than a silent +mismatch. + +A model trained with the RF temperature features, applied without `--temperature-dir`, says +so. The run still completes - the missing features are substituted the way training substitutes +a missing one - but two of them are then pinned to a value no real spectrum produced, and +nothing in the output would tell you: + +``` +WARNING: This model was trained with the RF temperature features, but no --temperature-dir was +given. They will be treated as missing for every spectrum. +``` + +The same is said per file when a directory was given but no CSV matches that run. + +The judgement call is whether "the same conditions" still holds. The instrument's +calibration drifts, which is the entire premise of the tool, so a model from three months +ago is not obviously applicable today. `mars qc` on the new files will say. + +--- + +## `mars verify` + +Round-trips a file through the passthrough writer applying a **null** correction - decode +and re-encode every m/z array without changing a value - then checks the result is +equivalent to the input. + +```bash +mars verify runs/one.mzML +``` + +| Option | Meaning | +|---|---| +| `-o, --output ` | Where to write the round-tripped copy (default `-verify.mzML` alongside the input) | +| `--keep` | Keep the round-tripped file (default: delete it) | +| `--threads ` | Worker threads (default: auto, one per logical processor) | +| `--check-offsets ` | Index offsets to spot check (default: all) | +| `-v, --verbose` | Verbose output | + +This separates the file-format work from the science. If `verify` passes, MARS can read and +rewrite this vendor's mzML faithfully, and any problem with a calibrated file is in the +model rather than the plumbing. If it fails, nothing else is worth investigating yet. + +Run it once per new instrument, conversion pipeline, or msconvert version. + +--- + +## `mars compare` + +Compares two mzML files on **decoded** m/z and intensity values. + +```bash +mars compare original.mzML corrected/original-mars.mzML +``` + +| Option | Meaning | +|---|---| +| `--validate` | Also check each file's index and checksum | +| `--max-report ` | Detail lines to print (default 10) | + +Use this rather than `cmp`. Byte comparison is meaningless here: two zlib implementations +produce different compressed bytes for identical data, so MARS's output is not +byte-identical across platforms even when every decoded value matches. That is +[documented behaviour](mzml-passthrough.md#binary-arrays), not a defect. + +## Global + +`mars --version` prints the version. `mars --help` prints the options above. diff --git a/docs/dotnet-port-spec.md b/docs/dotnet-port-spec.md new file mode 100644 index 0000000..654bb8d --- /dev/null +++ b/docs/dotnet-port-spec.md @@ -0,0 +1,1131 @@ +# MARS .NET 10 Port Specification + +**Status:** Draft +**Target repo:** `mars` (this repo, alongside the existing Python) +**Author:** M. MacCoss +**Last updated:** 2026-08-19 + +--- + +## 0. How to use this document + +This spec governs the port of MARS from Python to C# targeting .NET 10. The Python +implementation stays in this repo for the duration of the port and is the reference +oracle for correctness. It is removed only after the acceptance gates in +[Section 8](#8-acceptance-gates) pass. + +Sections marked **`[FILL]`** must be completed by transcribing from the Python source +before implementation starts. They are the parts of the system this spec cannot +specify from the outside, and they are also where a port most reliably goes wrong. + +> **All `[FILL]` sections have now been completed** from the Python source, and the +> sections they governed record what was implemented rather than what was intended. +> Where the port revised a decision the draft had already made, the revision is marked +> as such and says why. Section 10a records four defects the transcription turned up in +> the Python implementation; three of them affect files that have already been written. + +--- + +## 1. Goals and non-goals + +### Goals + +1. A C# implementation of MARS that produces recalibrated mzML files statistically + equivalent to the current Python implementation. +2. Deterministic output. Identical input produces a bit-identical m/z array in the + output file, on every platform, regardless of thread count. +3. No native dependencies. Pure managed code so the assembly drops into the managed + ProteoWizard tree without adding per-platform build artifacts. +4. A library boundary clean enough that MARS can later be invoked as a managed + msconvert filter, from Osprey, or as a standalone CLI, from one implementation. + +### Non-goals + +1. Byte-identical mzML output versus the Python implementation. The two use different + zlib implementations and different XML serialization paths, so compressed bytes will + differ. Equivalence is defined on **decoded m/z values**, not file bytes. See + [Section 8](#8-acceptance-gates). +2. Reproducing XGBoost's exact tree structure. The C# model is an independent + implementation of the same regularized objective, not a loader for XGBoost's + serialized model. Equivalence is defined on **post-correction error metrics**. +3. Correcting anything other than m/z. Intensity arrays, MS1 spectra, chromatograms, + and all metadata pass through untouched. +4. Rewriting the centroider. MARS consumes whatever spectra it is given. + +--- + +## 2. Decisions already made + +| Decision | Value | Rationale | +|---|---|---| +| Language | C# | ProteoWizard and msconvert are being ported to managed C#. Rust or C++ would strand MARS on the wrong side of that boundary. | +| Target framework | `net8.0`, opting into `net10.0` | **Revised.** See below. | +| Model implementation | Reuse `Osprey.ML.GradientBoostedTrees` | Already implements the XGBoost regularized objective (histogram split finding, Newton boosting, L1 + L2 leaf penalties, gamma, min_child_weight, subsampling) and is already deterministic by construction. | +| Model ownership | `Osprey.ML` remains the sole owner | `Osprey.FDR` needs it on net472, which MARS at net10 cannot supply. Two copies would drift silently in the split-finding code. | + +### Revision: target framework + +`Directory.Build.props` builds `net8.0` by default and takes the full matrix from a +single property: + +``` +dotnet build -p:MarsTargetFrameworks="net8.0;net10.0" +``` + +Three reasons for the default. A `net8.0` assembly executes unchanged on the .NET 9 and +.NET 10 runtimes, so nothing is given up at run time. It removes the forward-reference +problem recorded under "Recorded risk" below, since a net8 pwiz can reference a net8 +MARS. And it does not require every build machine to carry a .NET 10 SDK before MARS +will compile at all. + +Nothing in `MARS.Core` uses a net10-only API, so raising the floor later is the same +one-property change. +| mzML strategy | Passthrough | Byte-preserving modification of the existing file. Established in the Python implementation after psims and lxml-rewrite approaches produced files that broke DIA-NN and SeeMS. | +| Python removal | After acceptance gates pass | Not before. | + +### Recorded risk + +MARS targets `net10.0` while the pwiz port lands on `net8.0` first. .NET reference +compatibility is forward-only, so a `net10.0` assembly cannot be referenced from a +`net8.0` project. This is fine as long as MARS is consumed as a **process** +(CLI invocation) rather than as a library. If in-process integration with net8-era +pwiz becomes necessary before pwiz reaches net10, `MARS.Core` will need to +multi-target `net8.0;net10.0`. Keeping `MARS.Core` free of net10-only APIs costs +nothing now and preserves that option. + +--- + +## 3. Repository layout during the transition + +As built. The Python tree stays where it is rather than moving to `python/`, so that +`pip install -e .` and the existing test suite keep working untouched during the +transition. + +``` +mars/ +├── mars/ # existing Python implementation, frozen +├── tests/ # existing Python tests +├── dotnet/ +│ ├── MARS.sln +│ ├── Directory.Build.props # net8.0 by default, net10.0 opt-in +│ ├── global.json # rollForward latestMajor +│ ├── MARS.Core/ # domain types, matching, features, model, correction +│ ├── MARS.IO/ # mzML passthrough, library readers, managed SQLite +│ ├── MARS.OspreyML/ # compiles the vendored sources, nullable off +│ ├── MARS/ # CLI executable (mars.exe) +│ ├── MARS.Test/ # unit and contract tests +│ ├── third_party/Osprey.ML/ # vendored sources + UPSTREAM.json drift guard +│ └── scripts/sync-osprey-ml.ps1 +├── golden/ # NOT YET BUILT, see Section 9 +└── MARS-dotnet-port-spec.md # this file +``` + +`MARS.OspreyML` exists as its own project only so the vendored sources compile with +nullable reference types off, exactly as they do upstream. Splitting it out keeps the +vendored files byte-identical to their origin, which is what the hash guard checks. + +The Python tree is **frozen at port start**. Any change to Python feature extraction +after that point invalidates the golden fixtures and must be accompanied by +regenerating them. + +### `Directory.Build.props` + +Mirror the Osprey conventions: + +```xml + + + net10.0 + latest + enable + disable + true + true + University of Washington + Copyright (c) University of Washington 2026 + + +``` + +`InvariantGlobalization` matters: mzML attribute values and CLI arguments must parse +identically regardless of the host locale. All numeric formatting and parsing uses +`CultureInfo.InvariantCulture` explicitly. + +Source files carry the Apache 2.0 header used in `Osprey.ML`, including the +`AI assistance:` line where applicable. + +--- + +## 4. Consuming Osprey.ML + +MARS needs `GradientBoostedTrees` from `pwiz_tools/Osprey/Osprey.ML`. Three options, +in order of preference: + +1. **NuGet package.** Publish `Osprey.ML` from the pwiz build to GitHub Packages or an + internal feed. Cleanest boundary, versioned, no drift. Requires a packaging step + in the pwiz build that does not exist today. +2. **Git submodule** of pwiz with a sparse checkout of `pwiz_tools/Osprey/Osprey.ML`. + Works, but a submodule of a repository that large is unpleasant. +3. **Vendored copy** under `dotnet/third_party/Osprey.ML/` with a sync script and a + test that asserts the vendored file's SHA-256 matches a recorded upstream hash. + The test fails loudly when upstream changes, which converts silent drift into a + visible merge task. + +**Recommendation:** start with (3) to unblock, migrate to (1) when pwiz has a +packaging story. Do not start with a copy that has no drift guard. + +#### As implemented + +Option (3). `dotnet/third_party/Osprey.ML/` holds `GradientBoostedTrees.cs` verbatim +and a `XorShift64.cs` fragment, alongside `UPSTREAM.json` recording the pwiz commit and +a SHA-256 per file. Three things enforce it: + +- `MARS.Test.VendoredOspreyTest` fails when a vendored file stops matching its recorded + hash, which is what catches someone editing the copy instead of fixing it upstream. +- `dotnet/scripts/sync-osprey-ml.ps1 -PwizPath ` reports drift against a real pwiz + checkout, and with `-Apply` pulls the change down and rewrites the hashes. +- `XorShift64` is vendored as a fragment rather than a file, because upstream it lives + inside `LinearSvmClassifier.cs` next to MathNet and Osprey.Core dependencies. Its + guard is therefore semantic: a test asserts the output sequence for a fixed seed, which + is the property that actually has to hold. + +The upstream change is +[ProteoWizard/pwiz#4592](https://github.com/ProteoWizard/pwiz/issues/4592), on branch +`Skyline/work/20260819_osprey_gbt_regression`. It adds the objective, a `GbtModelData` +snapshot so a trained model can be persisted without reflecting over private state, and +the bit-identical parts of the Section 7 optimization list. The logistic path is +unchanged, asserted by a golden test over 1,925 scores across five fixtures including +NaN, infinity and constant columns. + +### Required change to Osprey.ML + +`GradientBoostedTrees` currently hard-codes binary logistic loss. Regression is added +as a second objective in **upstream Osprey.ML**, not forked into MARS. + +Everything in that class except the base score and the per-round gradient computation +is loss-agnostic: quantile binning, histogram split finding, the L1 soft-threshold and +L2 leaf weight, subsampling, and the flat node arrays all apply unchanged. +`ScoreSingle` already returns the raw additive margin with no link function, which for +squared error is the prediction itself. + +```csharp +public enum GbtObjective { LogisticBinary, SquaredError } +``` + +Add `public GbtObjective Objective = GbtObjective.LogisticBinary;` to `GbtParams` and +a `Train(double[][] x, double[] y, GbtParams p, double[] sampleWeight = null)` overload. + +Base score: + +```csharp +// SquaredError: weighted mean of y. LogisticBinary: existing log-odds. +double baseScore = p.Objective == GbtObjective.SquaredError + ? (tot > 0 ? sumWy / tot : 0.0) + : Math.Log(frac / (1 - frac)); +``` + +Per-round gradients: + +```csharp +// SquaredError: g = (f - y) * w, h = w. +for (int i = 0; i < n; i++) +{ + double wi = w != null ? w[i] : 1.0; + g[i] = (f[i] - y[i]) * wi; + h[i] = wi; +} +``` + +**Constraint:** the `LogisticBinary` path must remain byte-identical. The change is +gated on `p.Objective` with the existing code as the default branch, and an +`Osprey.Test` case asserts that an existing FDR training run produces a bit-identical +model before and after. + +#### `MinChildWeight` semantics change + +This is the one real trap. `MinChildWeight` thresholds the **summed hessian**, and the +hessian means different things under the two objectives: + +| Objective | `h_i` | `H` over a leaf | `MinChildWeight = 1.0` means | +|---|---|---|---| +| LogisticBinary | `p(1-p)`, at most 0.25, shrinking as the model sharpens | much less than the sample count | several samples, and more as boosting proceeds | +| SquaredError | `w_i`, exactly 1.0 unweighted | the sample count | **one sample** | + +The same applies to the leaf-stop condition `H < 2 * MinChildWeight` on line 232. + +The `GbtParams` defaults were tuned for the Percolator-replacement use case and are +**not** a valid starting point for MARS. Carry the hyperparameters over from the Python +XGBoost run directly, where `min_child_weight` under `reg:squarederror` already has +exactly this sample-count meaning. + +#### Python hyperparameters + +Transcribed from `MzCalibrator.__init__` and `MzCalibrator.fit` in +`mars/calibration.py`. The constructor sets four parameters explicitly and leaves +everything else at the XGBoost library default: + +```python +self.model = xgb.XGBRegressor( + n_estimators=self.n_estimators, # 100 + max_depth=self.max_depth, # 6 + learning_rate=self.learning_rate, # 0.1 + random_state=self.random_state, # 42 + n_jobs=-1, + objective="reg:squarederror", +) +``` + +| Parameter | Value | Source | +|---|---|---| +| `objective` | `reg:squarederror` | explicit | +| `n_estimators` | 100 | explicit, constructor default | +| `max_depth` | 6 | explicit, constructor default | +| `learning_rate` | 0.1 | explicit, constructor default | +| `random_state` | 42 | explicit, constructor default | +| `min_child_weight` | 1.0 | XGBoost default | +| `subsample` | 1.0 | XGBoost default | +| `colsample_bytree` | 1.0 | XGBoost default | +| `gamma` | 0.0 | XGBoost default | +| `reg_lambda` | 1.0 | XGBoost default | +| `reg_alpha` | 0.0 | XGBoost default | +| `max_bin` | 256 | XGBoost default | +| `tree_method` | `hist` | XGBoost 2.x default | +| `base_score` | fitted intercept | XGBoost 2.x fits it as the weighted mean of y | +| early stopping | **none** | an `eval_set` is passed but no `early_stopping_rounds`, so all 100 rounds run | + +`sample_weight` is the observed peak intensity divided by its mean +(`sample_weight / sample_weight.mean()`), so the weights average to 1. The +normalization is load-bearing under squared error, where the hessian IS the weight: +raw detector counts would put the summed hessian in the millions and make +`min_child_weight` meaningless. + +`validation_split=0.2` holds out 20% via `sklearn.model_selection.train_test_split` +with `random_state=42`. The held-out rows are only ever scored, never trained on. + +**Osprey.ML defaults do not transfer.** `GbtParams` ships `NTrees=200`, +`Subsample=0.8`, `ColSample=0.8`, `MaxBins=64`, tuned for the Percolator replacement. +MARS sets all twelve values explicitly in `CalibrationOptions` rather than inheriting +any of them. + +#### Missing value handling + +`GradientBoostedTrees` maps `NaN` to bin 0 (`BinOf`, line 298). XGBoost instead learns +a per-node default direction for missing values. If any MARS feature can be absent +(for example, a neighbor-density feature at the edge of a scan range where the window +is truncated), the two models will diverge in a way that is difficult to trace. + +**Requirement:** MARS feature extraction emits no `NaN` and no infinities. Every +feature has a defined value for every row, with truncated windows handled by an +explicit documented convention (zero count, or a separate indicator feature), not by +propagating `NaN`. A debug assertion in `MARS.Core` enforces this. + +--- + +## 5. Component specification + +### 5.1 `MARS.IO` — mzML passthrough + +Implements the passthrough contract already established by the Python version. The +non-negotiable rules, all of which are load-bearing for downstream tool compatibility: + +1. Write **indexed** mzML. DIA-NN fails silently on unindexed files. +2. Preserve `cvRef="MS"`. Never emit `cvRef="PSI-MS"`. +3. Preserve the Thermo nativeID format (`controllerType=0 controllerNumber=1 scan=NNNN`, + CV term `MS:1000768`) and all source file references. +4. Re-encode each modified binary array with **the same** compression and precision it + was decoded with. Read encoding per-array, never per-spectrum: m/z is typically + 64-bit while intensity is often 32-bit, and compression can differ between arrays in + one spectrum. +5. Update `encodedLength` on every modified array to the base64 **character** count. +6. Regenerate ``, ``, and the SHA-1 `` after + any modification. The checksum covers all bytes up to and including the + `` line. +7. Do not add or remove spectrum elements. Do not recompute derived CV terms + (base peak m/z, TIC) unless the correction actually invalidates them. + +#### Implementation approach: byte splice, not DOM round-trip + +The Python version parses to an lxml tree and re-serializes with `etree.tostring()`. +That works but is more invasive than necessary: the serializer can in principle perturb +attribute ordering, whitespace, and namespace declarations across the whole document. + +The C# implementation should instead treat the file as a byte stream and splice: + +1. Scan for `` element spans and record `(start, end)` byte offsets + along with the enclosing spectrum's `id`, `ms level`, and the array's CV params. + Use `XmlReader` for correctness of the scan; do not build a DOM. +2. Copy the input to the output verbatim, except that when the writer reaches a span + selected for modification, it emits a replacement built from the corrected array. +3. Everything outside the replaced spans is byte-identical to the input by construction. + +This is strictly more faithful than the Python path and removes an entire class of +serializer-induced compatibility bugs. It also streams, which the Python version does +not. + +#### Streaming and memory + +**Requirement:** never hold the input file in memory. MARS makes **two passes**: + +- **Pass 1** reads spectra, extracts features for training rows, and fits the model. +- **Pass 2** re-reads the file, applies the correction, and splices the output. + +Peak memory is bounded by the training feature matrix, not by file size. A 3 GB mzML +must process in under 2 GB of working set. + +#### Binary encoding notes + +- Decode: base64 (`Convert.FromBase64String`, tolerating embedded whitespace), then + zlib if declared. Use `System.IO.Compression.ZLibStream`, **not** `DeflateStream`: + mzML uses the zlib container with its 2-byte header and Adler-32 trailer. +- Reinterpret bytes as `double` or `float` via `MemoryMarshal.Cast`. + Assert `BitConverter.IsLittleEndian` at startup; mzML binary arrays are + little-endian by specification. +- .NET's zlib and Python's zlib produce different compressed bytes at the same nominal + level. This is expected and is why parity is defined on decoded values. + +#### Acceptance for `MARS.IO` alone + +A **null correction** (identity transform applied to every spectrum) must produce an +output file that: + +- opens in SeeMS without warnings, +- round-trips through `msconvert --mzML` without error, +- decodes to bit-identical m/z and intensity arrays versus the input, +- has a valid SHA-1 checksum and a correct index (verify by seeking to each recorded + offset and confirming a `The ordered subset a particular model was trained on. +public sealed class FeatureSet +{ + public MarsFeature[] Features { get; } + public int SlotOf(MarsFeature feature); // column index, or -1 + public string[] Names(); + public static FeatureSet FromNames(IReadOnlyList names); +} +``` + +A fixed struct was rejected because the active feature set is decided at fit time from +which columns carry data (see "Feature selection is dynamic" above), so the row width +is not known until then. + +Two extraction paths exist because the two contexts differ in what they know: + +```csharp +// Training: one row per matched library fragment, appended to a column store. +public sealed class FragmentMatcher +{ + public int MatchSpectrum(SpectrumRecord spectrum, TemperatureSet? temperatures, MatchTable table); +} + +// Correction: one row per peak, scored and subtracted in place. +public sealed class SpectrumCorrector +{ + public SpectrumCorrectionResult Correct( + SpectrumRecord spectrum, TemperatureSet? temperatures, + CorrectionWorkspace workspace, Span destination); +} +``` + +Correction is allocation-free per spectrum: `CorrectionWorkspace` owns every buffer and +grows only when it meets a larger spectrum than it has seen. Training appends into a +column-oriented `MatchTable` rather than a row of objects, because a nine-million-row +Astral plate would otherwise spend more memory on object headers than on data. + +The six neighbor windows are computed for ALL peaks in one monotone sweep rather than +per peak by binary search, since both window ends advance monotonically with m/z. Each +window's slice is summed directly rather than differenced out of a prefix sum, so the +result is bit-identical to the per-fragment path used during training. + +### 5.3 `MARS.Core` — model and correction + +```csharp +public sealed class MarsModel +{ + public static MarsModel Fit(double[][] features, double[] massError, MarsOptions o); + public double PredictError(ReadOnlySpan features); + public void Save(Stream s); + public static MarsModel Load(Stream s); +} +``` + +`Fit` delegates to `GradientBoostedTrees.Train` with `GbtObjective.SquaredError`. + +Correction is `corrected = observed - PredictError(features)`, with the sign convention +fixed by the label definition above: the label is `observed - theoretical`, so the +predicted error is subtracted. Get this wrong and the MAD roughly doubles instead of +halving, which is at least a loud failure. Measured on the reference cohort it halves: +0.0800 to 0.0464 Th on the written files. + +**Requirement:** the corrected m/z array must remain **strictly ascending**. A +per-peak correction can in principle reorder adjacent peaks. mzML consumers assume +sorted m/z arrays and some will produce silently wrong results otherwise. + +The Python implementation has no check of any kind: it writes `mz_array - corrections` +straight into the file. Chosen behavior for the port, since there was nothing to +inherit: + +| `--on-reorder` | Behavior | +|---|---| +| `clamp` (default) | Raise the offending peak to the next representable double above its predecessor. Strictly ascending, with the smallest perturbation that achieves it. | +| `revert` | Leave that whole spectrum uncorrected and count it. | +| `allow` | Write the values as-is. Present only for diagnosing how often it happens. | + +Every violation is counted and reported regardless of policy, so a model that reorders +peaks frequently is visible rather than silently patched. On the reference Stellar +cohort the count is **zero** across all 565,498 corrected spectra: corrections are two +orders of magnitude smaller than typical peak spacing, so clamping is a guard against a +pathological model rather than a routine occurrence. + +#### Model serialization + +Version the format from day one. A model file records: format version, MARS version, +the feature name list in order, the hyperparameters, the flat node arrays, the base +score, and the training run's identifier and row count. Loading a model whose feature +list does not match the extractor's current feature list is a hard error, not a +warning. + +### 5.4 `MARS` — CLI + +``` +mars recalibrate [-o ] [--model ] + [--save-model ] [--report ] + [--threads N] [--seed N] [--dry-run] +``` + +- `--dry-run` computes and reports metrics without writing an output file. +- `--report` emits per-spectrum and global before/after MAD and RMS, plus feature + importances, for the QC path. +- Exit codes: 0 success, 1 input error, 2 insufficient training data, 3 output + validation failure. +- All diagnostics to stderr, so stdout stays clean for piping. + +--- + +## 6. Determinism requirements + +MARS writes m/z values into files that will be reprocessed and compared. Determinism +is a correctness requirement, not a nicety, and it is a property that would be **lost** +by linking libxgboost, whose histogram tree method can vary with thread count. + +The invariants, following the Osprey determinism conventions: + +1. **Every floating-point accumulation happens in a fixed sequence.** The original + wording was "training is single-threaded", which is stronger than the property that + matters and would have made the Astral scale impractical. What is actually required + is that the model not depend on the thread count, and that is achieved by + parallelizing **across features only**: one thread owns a feature's histogram and + walks the node's rows in ascending order, so no summation order can drift. Histogram + subtraction, which would roughly halve the work per level, is deliberately NOT used, + because deriving a sibling histogram by subtraction changes the floating-point result. + `GbtParams.MaxDegreeOfParallelism` defaults to 1, leaving the Osprey.FDR path exactly + as sequential as it was; MARS opts in. +2. **Subsampling uses `XorShift64`**, seeded, never `System.Random`. +3. **Inference is embarrassingly parallel and carries no determinism risk.** Each + peak's prediction is independent with no cross-row accumulation, so parallelizing + pass 2 across spectra cannot change results. This asymmetry is the key to meeting + the performance targets: parallelize inference freely, never parallelize training. +4. **No `Dictionary` iteration order** reaches an output-affecting path. Anything + collected from a hash container is sorted by a stable key before use. +5. **Sorts have explicit tiebreakers.** Comparisons on doubles use a total order and + a secondary key (peak index) so equal values cannot reorder. +6. **No `NaN` reaches the model.** Enforced at extraction (Section 4). + +**Test:** run MARS twice on the same input with the same seed, decode both outputs, and +assert the m/z arrays are bit-identical. Run once with `--threads 1` and once with +`--threads 16` and assert the same. This test runs in CI on every commit. + +--- + +## 7. Performance requirements + +Measured on the reference Stellar cohort: 5 files, 6.0 GB of input, 565,498 MS2 +spectra, 57.0M MS2 peaks per file, 352,349 training rows, 20 features - 22 is the maximum +and this cohort has no temperature logs. Machine: 16 logical cores, NVMe. + +| Stage | Target | Measured | +|---|---|---| +| Pass 1 (read + match + extract) | I/O bound | 10 to 13 s per 1.2 GB file | +| Training | ≤ 60 s | 13 s (100 rounds, depth 6, 282k rows) | +| Pass 2 (infer + write) | ≤ 60 s | 24 to 38 s per file | +| Whole `calibrate`, 5 files | — | 229 s, of which 155 s is pass 2 | +| Null-correction passthrough | ≤ 2× msconvert copy | 6.9 s for 1.2 GB (176 MB/s) | +| Peak working set | ≤ 2 GB on a 3 GB input | bounded by the training matrix, not the file | + +Peak count per file turned out to be about 57M rather than the assumed 20M, and +training rows about 350k rather than 1 to 3M, so inference dominates and training does +not. That reverses the original expectation: the Section 7 optimization list matters +for the ASTRAL scale (9.1M training rows), not for Stellar. + +Streaming means memory is bounded by the largest single spectrum plus the training +matrix, never by file size. A 4.9 GB Astral file streams in the same working set as a +1.2 GB Stellar one. + +Inference cost is easy to underestimate: 20M peaks × 200 trees × depth 6 is roughly +2.4 × 10^10 node visits. Tree traversal is branch-heavy and cache-hostile. Mitigations, +in order of value: parallelize across spectra (free, see Section 6), keep the flat +node arrays hot (already the layout in `GradientBoostedTrees`), and consider whether +fewer or shallower trees give equivalent MAD. + +### Known optimizations available in Osprey.ML + +`GradientBoostedTrees` was sized for Percolator-scale input (roughly 100k rows). At +MARS scale the following are worth doing, all as **pure-throughput changes with no +behavioral effect**, each validated by asserting a bit-identical model before and +after: + +1. **Flatten the jagged arrays.** `double[][] x` and `byte[n][] bin` become + `double[n * nFeat]` and `byte[]`. At 2M rows the jagged form is 4M small arrays with + object headers, which is both heavy allocation and poor locality. +2. **Store `bin` column-major.** Histogram accumulation for feature *j* then walks + contiguous memory instead of striding across millions of separate arrays. Likely the + single largest win. +3. **Pool histogram buffers per depth level.** `BuildTree` currently allocates a fresh + `new double[maxBins]` pair for every feature at every node. +4. **Histogram subtraction.** Build the histogram for the smaller child only and derive + the sibling by subtracting from the parent. Roughly halves the work per level. +5. **In-place row partitioning.** Replace the per-node `List left/right` plus + `.ToArray()` with a pivot partition of a single index array. + +These belong upstream in `Osprey.ML`, where `Osprey.FDR` also benefits. + +--- + +## 8. Acceptance gates + +Three staged gates. Each isolates one failure mode. Do not proceed to the next until +the previous passes. + +### Gate A — feature parity + +The C# extractor must reproduce the Python feature matrix. + +**Harness.** `scripts/emit_golden.py` runs the frozen Python on the fixtures in +`golden/data/` and writes, per spectrum, the full feature matrix plus the peak index and +the mass-error label, to `golden/features/` as Parquet or TSV with full float64 +precision (`repr`-round-trippable, 17 significant digits). + +`MARS.Test` runs the C# extractor on the same fixtures and compares. + +**Tolerance.** Exact bit equality for counts, indices, and direct lookups (m/z, +intensity, RT, TIC). Relative tolerance 1e-12 for accumulated quantities (sums, means, +ratios), where Python and C# may differ in the last ULP from summation order. + +**The check that actually matters:** assert that no row's **quantile bin assignment** +changes under the tolerance. Feature differences only matter if they flip a tree +comparison, so this is the property with teeth. A test that passes on tolerance but +fails on bin assignment is a real bug. + +### Gate B — end-to-end feature validation + +Confirms the C# features are not just close but *usable* by the reference model, +without needing an XGBoost model loader in C#. + +1. C# writes its extracted feature matrix to disk. +2. `scripts/score_csharp_features.py` loads it and scores it with the **Python-trained + XGBoost model**. +3. Assert the resulting predictions match Python's own end-to-end predictions to 1e-9 + relative. + +This cleanly separates "the features are wrong" from "the model is different," which is +otherwise very hard to disentangle. + +### Gate C — model equivalence + +The C# GBT is an independent implementation, so predictions will not match XGBoost +exactly, and should not be expected to. Equivalence is statistical. + +Train the C# model on the same features and labels, apply the correction, and compare +against the Python result on a held-out set of runs: + +| Metric | Python (reference) | C# requirement | +|---|---|---| +| Post-correction MAD | 0.0435 Th | ≤ 1.05 × Python | +| Post-correction RMS | 0.0858 Th | ≤ 1.05 × Python | +| Median error centering | ~0 | \|median\| ≤ 0.005 Th | + +Baselines for context: uncorrected MAD 0.0800 Th and RMS 0.1189 Th, so the Python +implementation delivers a 46% MAD reduction and a 28% RMS reduction. + +#### Setting the tolerance + +The Python implementation is deterministic given the same inputs: `random_state=42` +fixes both the train/test split and XGBoost's sampling, and the reference cohort +reproduces its numbers exactly across the `output/`, `output-new/`, `output-repeat/` +and `output-test2/` directories in the repository. Its run-to-run variance on identical +input is therefore **zero**, and a variance-derived tolerance would be zero too. + +That makes run-to-run variance the wrong thing to set the gate from. What the gate has +to absorb is the difference between two independent implementations of the same +objective: different quantile cut points, a different train/test partition, and +different tie-breaking in split selection. The 5% figure is kept as an engineering +margin on that, not as a statistical bound. + +Measured on the reference Stellar cohort, all 20 features active (22 is the maximum; the two +temperature features need logs this cohort does not have): + +| Metric | Python | C# | Ratio | Gate | +|---|---|---|---|---| +| Matched fragments | 352,349 | 352,349 | 1.000 | — | +| Train/validation split | 281,879 / 70,470 | 281,879 / 70,470 | 1.000 | — | +| Pre-correction mean | −0.0134 Th | −0.0134 Th | 1.000 | — | +| Pre-correction median | −0.0082 Th | −0.0082 Th | 1.000 | — | +| Pre-correction std | 0.1180 Th | 0.1180 Th | 1.000 | — | +| Pre-correction MAD | 0.0800 Th | 0.0800 Th | 1.000 | — | +| Train MAE | 0.0622 Th | 0.0619 Th | 0.995 | — | +| Train RMSE | 0.0856 Th | 0.0853 Th | 0.996 | — | +| Validation MAE | 0.0629 Th | 0.0625 Th | 0.994 | — | +| Validation RMSE | 0.0864 Th | 0.0860 Th | 0.995 | — | +| **Post-correction MAD** | **0.0435 Th** | **0.0449 Th** | **1.032** | ≤ 1.05 PASS | +| **Post-correction RMS** | **0.0858 Th** | **0.0854 Th** | **0.995** | ≤ 1.05 PASS | +| **Median centering** | ~0 | **−0.0032 Th** | — | \|·\| ≤ 0.005 PASS | + +The pre-correction statistics agree to every reported digit. That is the result worth +noting: it means matching and feature extraction are faithful, and the only place the +two implementations diverge is inside the model, which is exactly where Section 1 +said they were allowed to. + +Post-correction MAD is the one metric where C# is worse, by 3.2%, and it sits inside +the 5% margin. RMS is very slightly better. + +#### The measurement that actually matters + +The table above compares what each implementation REPORTS, computed from its own +training-path features. That is not the deliverable. The deliverable is the corrected +mzML, so the honest test is to re-match the library against the WRITTEN files and +measure the mass error a downstream tool would actually see. + +Run as `mars qc` against each set of outputs with the same library and tolerance: + +| Measured on the written files | Uncorrected | Python-corrected | C#-corrected | +|---|---|---|---| +| Fragments matched | 352,349 | 358,320 | **358,334** | +| Mean delta m/z | −0.0134 Th | −0.0052 Th | **−0.0023 Th** | +| Median delta m/z | −0.0082 Th | −0.0046 Th | **−0.0025 Th** | +| Std delta m/z | 0.1180 Th | 0.0882 Th | **0.0872 Th** | +| MAD delta m/z | 0.0800 Th | 0.0472 Th | **0.0464 Th** | +| RMS delta m/z | 0.1188 Th | 0.0884 Th | **0.0872 Th** | +| Median delta ppm | −9.92 | −5.87 | **−3.14** | + +A paired measurement: both implementations run over the same cohort, on the same machine, with +the same library, tolerance and minimum intensity, and both sets of outputs scored by the same +`mars qc`. + +It was re-taken after the injection-time fix moved the C# column. Python's numbers are within +0.0001 Th of the run they replaced (0.0472 against 0.0471 MAD, 0.0882 against 0.0884 std), +which is the control worth having: it says the methodology is the same one, so the movement in +the C# column is a real change rather than a difference in how it was measured. + +Both implementations select the same 20 features here - Python logs +`Using 20 features` and drops only the two temperature features for want of logs. That is the +agreement the port was aiming at, and for a while C# did not have it: it was training on 5, +having switched off the injection-time and ion-population groups on data where they vary. + +The two corrected outputs are equivalent on every metric, with C# now modestly ahead on all of +them rather than level - most visibly on centering, where the median residual is −0.0025 Th +against −0.0046 Th. The match count rises in both because correcting the m/z pulls fragments +that sat outside the tolerance back inside it. + +Note also that both implementations' written files score slightly WORSE than their own +reported numbers (0.0872 on the written files against 0.0862 reported, for C#). That gap is +inherent to the design, +not a porting error: the model is trained with `fragment_mz` set to the library's +theoretical m/z and the neighbor windows anchored there, but at correction time neither +is available and the observed m/z stands in for both. Closing it would mean changing the +feature definition, which is out of scope for a port. + +#### Astral plate + +The second reference dataset: 3 runs, 14.4 GB of mzML, a 16.1 GB Skyline report +(67,119,180 rows), matched at ±10 ppm. + +| | Python | C# | +|---|---|---| +| Fragments matched | 9,145,497 | 4,211,731 | +| Pre-correction std | 0.0027 Th | 0.0028 Th | +| Post-correction std | 0.0027 Th | 0.0028 Th | +| Improvement | 2.0% | 1.5% | +| Train MAE | 0.0019 Th | 0.0020 Th | +| Train RMSE | 0.0027 Th | 0.0028 Th | +| Pre-correction MAD | — | 0.0014 Th | +| Post-correction MAD | — | 0.0013 Th | + +The match counts differ by roughly 2.2x, and that is expected rather than a discrepancy. +A Skyline report lists every transition once per replicate with an identical theoretical +`Product Mz`, so those rows are exact duplicates. MARS collapses them (1,462,106 +collapsed here) while the Python `groupby` keeps them all, and each duplicate produces a +duplicate match and a duplicate training row. Duplicating every row uniformly does not +change what the model learns; it triples the memory and the matching work. + +The substantive result is that both implementations agree the Astral data has very +little left to correct: 2.8 mTh is about 4 ppm at these masses, and neither +implementation moves it by more than 2%. That is a real finding about the data, and it +is worth knowing that MARS earns its keep on Stellar ion-trap data (46% MAD reduction) +and essentially not at all on an already well-calibrated Astral run. + +Throughput at this scale, on 16 logical cores: 97 s to read and index the 16.1 GB +report, 49 s to match each 4.7 GB run, 125 s to train on 3.37M rows by 20 features, +369 s in total. Peak working set about 4 GB, dominated by the training matrix. + +### Gate D — downstream compatibility + +The corrected output must be accepted by the full ecosystem, on real files, not just +the small fixtures. + +| Check | Status | +|---|---| +| The index is valid: seeking to every recorded offset lands on the element it names | **PASS**, all 114,638 offsets on a 1.2 GB file | +| The SHA-1 checksum validates | **PASS** (and the Python output FAILS, see 10a.1) | +| Null correction decodes to bit-identical m/z and intensity | **PASS**, 56,972,925 peaks | +| Mass accuracy improves on the written file | **PASS**, MAD 0.0800 to 0.0464 Th | +| DIA-NN completes a search with identifications within noise of the Python-corrected file | **outstanding** | +| SeeMS opens the file with no warnings | **outstanding** | +| Skyline imports it | **outstanding** | +| `msconvert` round-trips it | **outstanding** | + +The four outstanding checks need the tools themselves and cannot be automated here. +They are the remaining blockers on Gate D, and none of them should be assumed to pass +just because the structural checks do: the whole reason the Python implementation +settled on a byte-preserving passthrough was that psims and lxml-rewrite approaches +produced files that broke DIA-NN and SeeMS despite being valid mzML. + +Structural verification is available as a command, so these can be re-run at any time: + +``` +mars verify # null-correction round trip +mars apply --model m.json --validate ... # check every written file +mars compare a.mzML b.mzML --validate # decoded-value diff between two outputs +``` + +--- + +## 9. Milestones + +| # | Deliverable | Gate | Status | +|---|---|---|---| +| 1 | Repo scaffolding, `Directory.Build.props`, CI, Osprey.ML vendoring with drift guard | builds clean | **done** | +| 2 | `MARS.IO` passthrough with null correction | Section 5.1 acceptance | **done**, except the downstream-tool checks in Gate D | +| 3 | Golden fixtures emitted from frozen Python | fixtures checked in | not done, see below | +| 4 | `GbtObjective.SquaredError` in upstream Osprey.ML | logistic path bit-identical | **done**, PR pending review | +| 5 | `MARS.Core` feature extraction | Gate A, Gate B | equivalent evidence, see below | +| 6 | End-to-end fit-and-correct, CLI | Gate C | **done** | +| 7 | Performance work (Section 7 optimizations) | targets met, models bit-identical | **done** for the bit-identical subset | +| 8 | Real-data validation | Gate D | Stellar done; Astral and downstream tools outstanding | +| 9 | Python decommission | see Section 10 | not started | + +### On milestones 3 and 5 + +Gates A and B were specified as a golden-fixture harness: emit Python's feature matrix, +compare the C# matrix row by row, then score the C# features with the Python model. + +That harness was not built, and the reason is worth recording rather than hiding. The +end-to-end run produced a stronger result than the gates were designed to detect: on the +reference cohort the C# implementation matches **exactly 352,349 fragments**, the same +count as Python, and reports pre-correction mean, median, standard deviation and MAD +that agree with Python's to every digit in its report (−0.0134, −0.0082, 0.1180, 0.0800). + +Reproducing the match set exactly means the precursor and RT windowing, the +most-intense-peak rule, the tolerance handling and the label are all faithful. Agreeing +on the moments of the label distribution to four decimals across 352,349 rows means the +same rows were selected. What that evidence does NOT cover is the 14 injection-time +features, which influence the model but not those statistics. + +So the honest position is: Gate A's intent is satisfied for matching and the label, and +unverified for the neighbor-density features. The fixture harness remains the right way +to close that gap and should be built before the Python is deleted, per Section 10. + +--- + +## 10. Python decommission + +The Python implementation is removed only when **all** of the following hold: + +1. Gates A through D pass. +2. The C# implementation has processed at least one full production cohort and the + results have been reviewed. +3. The golden fixtures in `golden/` are retained **permanently**, with a note in + `golden/README.md` recording that they were generated by the Python implementation + at commit ``. They remain the regression suite after the Python is gone. +4. `scripts/emit_golden.py` is retained (it will no longer run, but it documents + exactly how the fixtures were produced). +5. The final Python commit is tagged `python-final` so it stays reachable. + +Delete the code, keep the evidence. + +--- + +## 10a. Defects found in the Python implementation + +Transcribing a system line by line is an audit, and this one turned up four defects. +Each is recorded here with how it was confirmed, because three of them change the +contents of files that have already been distributed. + +### 10a.1 The written `fileChecksum` is invalid + +`write_calibrated_mzml` hashes everything up to the start of the two-space indent that +precedes ``: + +```python +checksum_content = modified_bytes + index_xml.encode("utf-8") + offset_line.encode("utf-8") +sha1 = hashlib.sha1(checksum_content).hexdigest() +``` + +The mzML convention is to hash every byte up to **and including** the `` +opening tag. Confirmed empirically: a pwiz-written input reproduces its recorded digest +only under the inclusive convention, and a MARS-written output reproduces its recorded +digest only under the Python one. + +| File | Recorded | Inclusive (spec) | Exclusive-of-indent (Python) | +|---|---|---|---| +| pwiz input | `ef83e3cb…` | **match** | no | +| Python MARS output | `6ff76251…` | no | **match** | + +Every mzML the Python implementation has written carries a checksum that fails +validation. Most consumers never check it, which is why this went unnoticed. +The C# writer uses the inclusive convention, and `mars verify` and `mars apply +--validate` check it. + +### 10a.2 `absolute_time` is re-based for training but not for correction + +`cli.py` subtracts the earliest acquisition across the cohort before fitting: + +```python +combined_matches["absolute_time"] = combined_matches["absolute_time"] - min_absolute_time +``` + +so the model learns on values in roughly 0 to 8,400 seconds. `write_calibrated_mzml` +then feeds the raw Unix timestamp back in: + +```python +absolute_time = acquisition_start_time + meta["scan_time"] * 60.0 # about 1.73e9 +``` + +Every inference row therefore lands above the largest value the model ever saw, and the +feature collapses to whichever branch the top bin leads to. The C# implementation stores +the offset in the model file and subtracts it again at correction time. + +### 10a.3 The TIC features are computed from different quantities in the two paths + +Training uses the summed decoded intensity array (`tic = float(np.sum(intensity_array))` +in `read_dia_spectra`); correction uses the `MS:1000285 total ion current` cvParam +(`"tic": spec.get("total ion current", 0.0)`). On Thermo centroided data these differ, +so `log_tic` and `tic_injection_time` are on different scales in the two paths. The C# +implementation uses the summed array in both. + +Both 10a.2 and 10a.3 are reproducible with `--python-compat`, so the two behaviors can +be compared on the same input. + +### 10a.4 A blib without peak annotations trains on the wrong quantity + +`load_blib` creates a `Fragment` for every peak in a reference spectrum, and for peaks +that carry no annotation it uses the stored m/z directly. A blib stores the **observed** +m/z of the reference spectrum, so matching against it measures the difference between +two runs' calibration errors rather than an absolute mass error. + +On `example-data/Stellar-HeLa-GPF.blib`, which has an empty `RefSpectraPeakAnnotations` +table, this yields 7.5M pseudo-fragments and 7.9M matches from a single file, and a model +that reduces the spread by 2.2%. MARS refuses this input and names the alternatives +rather than producing a model from it. + +A related point: even when annotations exist, `load_blib` recalculates b and y fragment +m/z from the STRIPPED sequence via `calculate_fragment_mz(stripped, ...)`, discarding +modifications. Every fragment of a modified peptide that spans the modified residue then +gets a theoretical m/z that is wrong by the modification mass. The C# reader applies the +per-position deltas from the blib's own `Modifications` table. + +--- + +## 11. Open questions + +Still open after the port: + +1. **Where does MARS sit in the pipeline?** Before or after centroiding? If MARS + assumes centroided input, that is a documented precondition and the CLI should + detect and reject profile-mode spectra rather than producing nonsense. + *Still open.* Neither implementation checks. `MS:1000127 centroid spectrum` and + `MS:1000128 profile spectrum` are both parsed already, so the check is cheap once + the intended precondition is decided. +2. **Does the correction need to be recorded in the output file?** Adding a + `` entry naming MARS and its version is standard practice and makes + corrected files self-describing. It does perturb byte offsets, but the index is + regenerated anyway. Recommend yes. + *Not implemented, deliberately.* Neither implementation records anything, so a + corrected file is indistinguishable from an uncorrected one except by its contents. + It should be added, but not in the same change as the passthrough: every other + modification MARS makes is confined to a `` element inside a spectrum, and + this one would edit `` in the file header, which is the one region + the passthrough currently guarantees it never touches. Worth doing as its own change, + with its own round-trip test, once the passthrough has been through Gate D. +3. **Should the model be embedded in the output?** A corrected file that carries the + model that produced it is fully reproducible. Probably as a `` reference + to a sidecar rather than inline. + *Not implemented.* Half of it is already there: the model is a versioned, readable + JSON file recording the format version, MARS version, ordered feature names, every + hyperparameter, the acquisition-time offset, the flat node arrays and the training + row counts. What is missing is the pointer from the mzML back to it, which is the + same header-splicing problem as question 2 and should be settled with it. +4. **Centroider language.** Now that all of pwiz is going managed, the sparse + deconvolution and non-negative LASSO centroider should presumably also target C# + rather than Rust. Out of scope here, but it is the same toolchain decision and + should be settled consistently. diff --git a/docs/model.md b/docs/model.md new file mode 100644 index 0000000..1f707ef --- /dev/null +++ b/docs/model.md @@ -0,0 +1,541 @@ +# The model + +MARS predicts one number per peak: the mass error, in Thomsons. The corrected value is +`observed - PredictDelta(features)`. Everything else in the tool exists to produce good +training rows for this, or to write the answer back out without breaking the file. + +This page covers the model itself. For how the training rows are produced, see +[the algorithm](algorithm.md). + +## Why gradient boosted trees + +The error is not a smooth function of anything. It has a broad dependence on m/z, a drift +over acquisition time, and a strong dependence on how crowded the ion trap was when the +spectrum was taken - which shows up as sharp, interacting effects between injection time, +total ion current and the population of ions in neighbouring m/z windows. + +A polynomial in m/z, which is the traditional approach, captures the first of these and +none of the rest. Boosted trees capture interactions without being told which ones to +look for, tolerate features on wildly different scales without normalization, and do not +extrapolate wildly outside the training range - a real virtue when the correction is being +applied to millions of peaks that may sit anywhere. + +The cost is that the model is opaque, which is why the QC report leads with permutation +importance and per-feature trends rather than the fitted parameters. + +## Where the implementation comes from + +The boosting code is **`Osprey.ML.GradientBoostedTrees`** from +[ProteoWizard](https://github.com/ProteoWizard/pwiz), vendored into +`dotnet/third_party/Osprey.ML/` rather than reimplemented. + +The regression objective MARS needs was contributed upstream rather than forked, so one +boosting implementation is maintained across the lab's C# tools instead of one per tool. +Osprey uses the binary-logistic path for FDR; MARS uses the squared-error path. The split +finding, histogram construction, regularization and determinism machinery are shared. + +The vendored copy is byte-for-byte identical to upstream, and `MARS.Test` hashes it against +`UPSTREAM.json` on every build. Editing the local copy fails the build. Fix bugs upstream +and re-sync with `scripts/sync-osprey-ml.ps1`. + +> The bit-identity of the shared path was verified when the regression objective was added: +> the same five logistic fixtures produce the same 1,925 scores, hashing to +> `242558FF0A6FCD3A5C34BE0A57BD42848A706853BD51E998E21C7B8856852A22`, before and after the +> change and at 1, 4 and 16 threads. Adding regression did not perturb Osprey. + +## The objective + +The regularized objective from [Chen & Guestrin +2016](https://arxiv.org/abs/1603.02754), the same one XGBoost optimizes. + +Each round fits a tree to the gradient and hessian of the loss at the current prediction. +Under squared error with sample weight `w`: + +``` +gradient g = w * (prediction - y) +hessian h = w +``` + +The hessian being exactly the weight has a consequence worth stating plainly: **a summed +hessian is a summed weight, and with unit weights it is a sample count.** That is why +`min_child_weight` reads as "minimum samples per leaf" here, and why the same parameter +means something quite different on the logistic path, where the hessian shrinks as the +model sharpens. + +A candidate split is scored by the gain it produces: + +``` +gain = 0.5 * ( GL^2/(HL+lambda) + GR^2/(HR+lambda) - (GL+GR)^2/(HL+HR+lambda) ) - gamma +``` + +and taken only if that exceeds `gamma`. Leaf values are the same expression's minimizer, +with the L1 penalty `alpha` applied by soft-thresholding the summed gradient. + +The base score is the **weighted mean of the labels** - the average mass error over the +training rows - so the first tree starts from the cohort's overall bias rather than zero. + +## Splits are found on histograms + +Fitting a tree by sorting every feature at every node would be hopeless at nine million +rows. Instead each feature is quantile-binned once, up to `max_bins` (256) bins, and every +value is replaced by a byte bin index. Split finding then walks a histogram of summed +gradients and hessians per bin, which is a fixed cost per node regardless of how many rows +land in it. + +The practical consequence: **a threshold is a bin edge, not an arbitrary value.** With 256 +quantile bins the resolution is fine enough that this is invisible in the output, but it +does mean two features with identical quantiles produce identical candidate splits, and +tie-breaking then decides between them. + +## Hyperparameters + +The defaults are XGBoost's defaults, deliberately. The Python implementation used +`XGBRegressor` with only four parameters set, leaving the rest at library defaults, so +matching them is what makes the two implementations comparable at all. + +| Option | Default | What it does | +|---|---|---| +| `--n-estimators` | 100 | Boosting rounds | +| `--max-depth` | 6 | Maximum tree depth | +| `--learning-rate` | 0.1 | Shrinkage applied to each tree's contribution | +| `--seed` | 42 | Seeds subsampling and tie-breaking | +| `--validation-split` | 0.2 | Held-out fraction; 0 trains on everything | +| - | `min_child_weight` 1 | Minimum summed hessian per leaf | +| - | `subsample` 1.0 | Row sampling per round | +| - | `colsample_bytree` 1.0 | Feature sampling per tree | +| - | `gamma` 0 | Minimum gain to take a split | +| - | `reg_lambda` 1.0 | L2 penalty on leaf weights | +| - | `reg_alpha` 0 | L1 penalty on leaf weights | +| - | `max_bins` 256 | Quantile bins per feature | + +The ones without a flag are not exposed on the command line. They are recorded in the model +file, so a model always says what produced it. + +There is rarely a reason to change any of this. The error surface is smooth enough that 100 +shallow trees fit it comfortably, and the failure mode that actually bites is not +underfitting - it is training on rows whose "theoretical" m/z was never theoretical. See +[spectral libraries](spectral-libraries.md). + +## Rows are weighted by intensity + +Each training row is weighted by the matched peak's intensity, normalized so the weights +average 1. + +The reasoning is that a strong peak has a better-determined centroid. A peak just above the +`--min-intensity` floor may be a few ions, and its apparent m/z carries counting noise that +has nothing to do with the instrument's calibration. Weighting by intensity lets the model +listen to the peaks that actually know where they are. + +**The normalization is not cosmetic.** `reg_lambda` and `min_child_weight` are thresholds on +summed hessians, which under squared error are summed weights. Feeding raw detector counts, +which run to 10^5 and beyond, would make `reg_lambda = 1` a rounding error and +`min_child_weight = 1` meaningless. Both implementations normalize to mean 1 for this +reason. + +## Mismatched peaks, and the second pass + +Matching takes the most intense peak inside the tolerance window, and sometimes that peak is +not the fragment. Those rows carry a delta that is not a mass error at all - it is the +distance to whatever ion happened to be there - and squared error is exactly the loss that +lets them pull the fit around. + +They are identifiable as a population. On the reference Stellar run, the 7.6% of rows with a +residual beyond 0.15 Th are: + +| | core 92.4% | tail 7.6% | +|---|---|---| +| median peak intensity | 3,818 | **1,064** | +| median fragment ions in the spectrum | 39.1 | **10.8** | +| raw \|delta\| within 0.06 Th of the window edge | 4.8% | **34.9%** | + +Weak peaks, in sparse spectra, sitting against the edge of the matching window. That is what +a wrong assignment looks like. The core alone has MAD 0.0389 Th and a near-Gaussian shape +(std/MAD 1.56); including the tail the ratio is 1.95, which is the tail announcing itself. + +So MARS fits once, then fits again with those rows removed - `--robust trim`, the default. +Two details matter: + +- **The scale is derived from the median absolute deviation**, not the standard deviation. + The outliers being looked for would inflate a standard deviation enough to hide + themselves. +- **Only training rows are trimmed.** Held-out rows are always scored in full. Dropping the + hard cases from the measurement as well would improve the reported number without + improving anything real. + +`--robust-sigma` sets the threshold. Measured out-of-fold, held-out rows scored in full: + +| `--robust-sigma` | out-of-fold MAD | rows trimmed | +|---|---|---| +| 0 (off) | 0.0445 Th | - | +| 2 | 0.0442 Th | 11.5% | +| 2.5 | 0.0442 Th | 6.4% | +| 3 (default) | 0.0442 Th | 3.7% | +| 4 | 0.0443 Th | 1.4% | + +Small, but real, and notably **flat across the threshold**, which says it is removing a +genuine contaminant rather than tuning against the folds. The default takes the full benefit +while discarding the least data. + +### Why trimming rather than a robust loss + +The textbook answer is a robust loss - Huber rather than squared error - which softens the +tail instead of cutting it. MARS implements that as `--robust huber`, and it does not need a +new objective: Huber's gradient is the residual clipped to the threshold, +`clip(r, +/-d) = r * min(1, d/|r|)`, so squared error on weights `w * min(1, d/|r|)` produces +exactly it. One extra pass of the existing path is a Huber fit. + +Measured, with the fold-to-fold spread alongside so the differences can be read against the +noise in them: + +| | 600-700 window | 400-500 window | +|---|---|---| +| `--robust none` | 0.0445 Th | 0.0523 Th | +| `--robust trim` | 0.0442 Th | 0.0505 Th | +| `--robust huber` | 0.0443 Th | 0.0518 Th | +| *fold-to-fold spread* | *0.0005 Th* | *0.0021 Th* | + +Read carefully, this does **not** say Huber is worse. On the 600-700 window the two differ +by 0.0001 Th, one part in four thousand - noise. On the 400-500 window trim is ahead by +0.0013 Th, which is real enough to notice but still smaller than the 0.0021 Th spread between +folds. Trim is the safer default on the evidence available; Huber is not refuted by it. + +What the numbers do suggest is a mechanism worth remembering. Huber assumes an outlier is an +extreme measurement of the *right* quantity and softens it in proportion - at three robust +sigma it still leaves such a row **79% of its weight on average**. A mismatched peak is not +an extreme measurement of the fragment's mass error; it is an accurate measurement of a +*different ion*, and its label carries no information about the quantity being fitted. On the +smaller, noisier window - where there is less real signal to outvote it - leaving four fifths +of that influence in appears to cost something. + +That reasoning also predicts where Huber should do better: a larger cohort, where the +contaminated rows are outnumbered and the cost of discarding real-but-extreme rows starts to +matter more than the cost of keeping mislabelled ones. Worth revisiting on more data rather +than treating the default as settled. + +> Do not reach for a tighter `--tolerance` instead. A wide window is a robustness property: +> a faster ion-trap scan rate gives worse mass accuracy, and a tolerance tuned to one cohort +> would silently under-match those runs. The window should stay wide enough for the worst +> instrument you intend to support, and the contaminated rows dealt with afterwards. + +## Which features the model gets + +Not a fixed list. The feature set is chosen from what the data actually supports: + +- If no MS2 spectrum carries an ion injection time, all fifteen features that count ions are + dropped rather than filled with zeros. +- If it is carried but never changes across the matched rows, only `injection_time` and + `tic_injection_time` go. The thirteen features it merely scales are kept - see + [the algorithm](algorithm.md#features-are-selected-not-assumed) for why, and for why that + question is settled over the whole column rather than a sample of it. +- Temperature features appear only when the matching CSVs were supplied. +- A row with any undefined feature value is dropped, not imputed. + +The model file records the ordered feature names it was trained on, and loading a model +whose features do not match what the extractor produces is a hard error. A silently +misaligned feature vector would produce plausible numbers and wrong corrections, which is +the worst possible failure mode for this tool. + +The 22 features and what each one is for are documented in +[the algorithm](algorithm.md#step-2-feature-extraction). + +## Determinism + +Identical input produces a bit-identical model and bit-identical output at any thread +count. This is a hard requirement, not a nice property: MARS writes m/z values into files +that get reprocessed, compared, and searched, and a tool whose output depends on the thread +count makes every downstream comparison unreliable. + +It is achieved by construction: + +- Parallelism runs **across features only**, never across rows. Each thread owns whole + feature columns, so no float accumulation is split across threads. +- Every float accumulation - histograms, leaf gradients and hessians - runs in a fixed + order determined by the data, not by scheduling. +- Subsampling and tie-breaking use a seeded XorShift64 PRNG, drawn in a fixed sequence. +- Train/validation splitting shuffles a seeded permutation and then **re-sorts each side + into ascending row order**, so downstream accumulation order is a function of the data + rather than of the shuffle. + +CI enforces this as its own job, separate from the rest of the test suite, so a failure is +unmistakable, and a unit test saves two independently fitted models and compares the files +byte for byte. + +Verified end to end on real data: the same `mars calibrate` invocation run twice on a +1.2 GB Stellar file, plus a third run at `--threads 1`, produce a byte-identical +`mars_model.json` (4,194,948 bytes), a byte-identical `mars_qc_summary.txt`, and a +byte-identical 1,510,067,312-byte corrected mzML. The per-fold cross-validation figures +match to every printed digit across thread counts. + +The one thing that is *not* bit-identical is the compressed bytes of the output file: +different platforms ship different zlib builds. Decoded values are identical. Use +`mars compare`, not `cmp`. See [mzML passthrough](mzml-passthrough.md#binary-arrays). + +## Cross-validation + +By default MARS trains **five models, one per fold, with folds split by peptide**, and +reports what each scored on the peptides it did not see. + +### Why the split has to be by peptide + +This is the part that matters most, and it is easy to get wrong. + +A peptide's fragments recur across hundreds of spectra, always with the same theoretical +m/z - and `fragment_mz` is a model feature. Split rows at random and the same peptide lands +on both sides of the boundary, so the model can memorize "this exact m/z has that error" +instead of learning anything about the instrument. The held-out number then measures recall +rather than generalization, and it comes out flattering. + +Splitting by peptide closes that route. Every reported number comes from a model that never +saw the peptide it is scoring, which is what makes it an estimate of performance on data +MARS was not trained on. + +Folds are assigned by sorting the distinct peptides and dealing them round-robin. No random +seed is involved, so the split is reproducible from the input alone, and every fold gets an +equal number of peptides. This follows Osprey's Percolator implementation, which splits its +folds the same way (`PercolatorSampling.CreateStratifiedFoldsByPeptide`). + +### Calibration is in-sample, and that is not a problem + +The model that corrects the data is fitted to **all** of it. Nothing is held back. + +That deserves stating plainly, because "trained on the data it is applied to" sounds like a +mistake. It is not. It is what mass calibration has always been: you measure species whose +masses you know in the run in front of you, and you correct the axis from them. Nobody +holds out half their lock masses. + +Two properties keep it honest: + +- **The correction moves a peak onto a fitted surface, not onto its theoretical m/z.** The + model never sees peptide identity, and it predicts from about twenty spectral features + over hundreds of thousands of rows. A hundred trees of depth six has nowhere near the + capacity to store a per-peak correction even if it wanted to. +- **The residual is measured, not assumed.** On the reference Stellar run the fit leaves + 0.0431 Th on the data it was fitted to and 0.0445 Th on peptides it never saw - a gap of + 0.0014 Th, 3% of the error being corrected. If the model were memorizing, that gap would + be wide. + +So the cross-validation is not there to catch cheating. It answers different questions. + +### What cross-validation is for here + +1. **Is the structure real?** If the out-of-fold figure is close to the in-sample one, the + surface describes the instrument rather than the particular peptides that happened to be + identified. If it is far off, the fit is thin. +2. **What will `mars apply` achieve?** Reusing a model on other files genuinely is + out-of-sample, and the out-of-fold figure is the honest estimate for it. +3. **Is there enough data?** The reference cohort's 400-500 window has 14,432 matches over + about 300 peptides and reports a gap of 0.0100 Th against a corrected error of 0.0523 Th + - roughly 20%. The 600-700 window, with ten times the matches, reports 3%. Same tool, + same settings, honestly different answers. + +Both numbers appear in the report, labelled: + +``` +After Calibration (these files, corrected): MAD 0.0431 Th +Expected on data not used to fit: MAD 0.0445 Th +``` + +The first is what the corrected files will look like when re-matched. The second is what to +expect from `mars apply` elsewhere. Quoting only the first would overstate transferability; +quoting only the second would understate what the correction actually did. + +### What it costs + +One extra training round per fold, and nothing at correction time, because the applied +model is an ordinary single fit: + +| | one 1.47 GB Stellar file | +|---|---| +| `--cv-folds 0` | 52 s | +| `--cv-folds 5` (default) | 66 s | + +An earlier design applied the fold models as an ensemble, so that the object shipped was +literally the object measured. That is what Osprey's Percolator does, and it is exact for +trees as well as for linear models - a boosted ensemble's score is linear in its trees, so +keeping every tree and dividing each leaf by K reproduces the average of K models to the +last bit. It was dropped because it makes correction five times slower for no benefit here: +the ensemble's members each saw four fifths of the peptides, and if in-sample calibration is +legitimate then the best surface to calibrate with is the one fitted to everything. + +### What it reports + +Per fold and pooled: median absolute residual, RMS, standard deviation, the reduction in +median absolute error, and Pearson r between predicted and observed error. Plus the +standard deviation of each across folds, which is what says whether a single held-out +number was luck. + +On one Stellar run, 146,515 fragments over 2,966 peptides: + +``` + fold rows MAD Th RMS Th reduction Pearson r + 1 29,542 0.0452 0.0860 44.6% 0.6904 + 2 29,590 0.0442 0.0856 44.7% 0.6872 + 3 29,386 0.0443 0.0856 44.1% 0.6899 + 4 28,902 0.0448 0.0860 44.3% 0.6889 + 5 29,095 0.0445 0.0861 44.3% 0.6845 + + pooled out-of-fold: MAD 0.0446 Th, RMS 0.0858 Th, r 0.6883 + spread across folds: MAD 0.0004 Th + in-sample MAD 0.0431 Th; optimism 0.0015 Th +``` + +**Optimism** is the gap between what the model scores on rows it was built from and what it +scores on unseen peptides. Here it is 0.0015 Th, about 3% of the error being corrected, so +the model is generalizing rather than memorizing. A large gap would mean the opposite, and +would say that any in-sample figure is not worth quoting. + +The before/after numbers everywhere else - the QC summary, the figures, the verdict line - +use these out-of-fold predictions. No optimistic number is reported anywhere as though it +were the result. + +### On high-resolution data + +The same machinery runs unchanged on Orbitrap Astral data with `--tolerance-ppm 10`, and the +answer it gives there is worth reading because it is mostly "there is little to do": + +``` +1,408,902 matched fragments over 81,184 peptides, 5 folds + + before MAD 0.0014 Th std 0.0029 Th median -0.0009 Th + after (these files) MAD 0.0013 Th std 0.0028 Th median -0.0001 Th + expected on new data MAD 0.0013 Th + gap 0.0000 Th fold spread 0.0000 Th + Pearson r 0.144 +``` + +Three things to take from it. + +**The improvement is small and real.** 5.6% off the median absolute error, 1.2% off the +spread. What the model mostly finds is a constant offset: the median error moves from +-0.0009 Th to -0.0001 Th, about 89% of a -1.5 ppm bias at m/z 600 removed. The rest of the +Astral error is not systematic in anything MARS measures. + +**Pearson r of 0.144, against 0.69 on Stellar.** That is the honest signal that there is +little structure to find. On an ion trap the space-charge effects dominate and the model +tracks them closely; on an Astral the instrument has already removed most of what is +predictable, and what remains is close to random. A tool that reported a large improvement +here would be describing noise. + +**The gap is zero.** With 1.4 million rows over 81 thousand peptides, in-sample and +out-of-fold agree to four decimal places, and the folds agree with each other to four +decimal places. So the small improvement is trustworthy, not an artifact - which is exactly +the question cross-validation exists to answer, and the answer is more useful here than on +data where the correction is obvious. + +The run takes about seven minutes end to end for one 4.9 GB file, of which a minute is +reading the 16.1 GB, 67-million-row plate report. + +### When there are too few peptides + +Cross-validation needs at least as many distinct peptides as folds, and in practice many +more. MARS refuses rather than producing folds of one or two peptides, and says how many it +found. `--cv-folds 0` falls back to a single fit with a held-out split - which is also +split by peptide, for the same reason. + +## Permutation importance + +The QC report ranks features by permutation importance: shuffle one feature's values across +the validation rows, re-score, and measure how much the error degrades. Normalized to sum +to 1. + +This is reported rather than split counts because split counts mislead. A feature with many +distinct values collects splits simply by offering more places to cut, whether or not those +cuts help. Permutation importance measures what the model would lose without the feature, +which is the question actually being asked. + +A feature near zero is carrying no weight and could be dropped. On the reference Stellar +cohort the two RF temperature features score below 0.01 each, which is the evidence behind +the advice that they are worth having when the logs exist and not worth chasing when they +do not. + +## The model file + +Versioned JSON, written to `mars_model.json`. Not interchangeable with the Python +implementation's pickled XGBoost booster; retrain rather than convert. + +```jsonc +{ + "formatVersion": 2, + "marsVersion": "26.1.0", + "featureNames": ["precursor_mz", "fragment_mz", ...], // ordered; must match at load + "absoluteTimeOffset": 1733158420.0, // seconds, see below + "options": { "nEstimators": 100, "maxDepth": 6, ... }, + "model": { "baseScore": ..., "objective": "SquaredError", "featureCount": 20, + "feature": [...], "threshold": [...], "left": [...], "right": [...], + "leaf": [...], "treeRoot": [...] }, + "training": { "rowsMatched": ..., "rowsTrain": ..., "trainMae": ..., ... }, + "crossValidation": { "folds": 5, "groups": ..., "outOfFoldMad": ..., "foldMad": [...], ... } +} +``` + +`formatVersion` has to match exactly; a model written by a different format version is +refused rather than read on a best guess. Version 2 added the `crossValidation` section, +which carries the out-of-fold numbers the report is drawn from. + +The trees are stored as flat parallel arrays rather than nested objects: one entry per +node, with `treeRoot` indexing where each tree starts. A hundred trees of depth six is +several thousand nodes, and a nested representation triples the file size for no benefit. + +`absoluteTimeOffset` is the part most worth understanding. Acquisition time is re-based to +the earliest matched spectrum before training, so the feature starts near zero. The offset +therefore has to travel with the model and be subtracted again at correction time. The +Python implementation re-bases for training but feeds raw Unix timestamps back in when +writing, so every inference row lands far above the largest value the model ever saw and +the feature collapses to a single branch. That is one of the four defects the port +deliberately does not reproduce; see +[port spec section 10a](dotnet-port-spec.md#10a-defects-found-in-the-python-implementation). + +## How close is this to XGBoost? + +Indistinguishable, once both are measured honestly. Both were trained on identical rows - +the features are verified bit-identical, see [parity](python-parity.md) - with identical +hyperparameters, identical weighting, and the same peptide-grouped fold split, on 146,515 +fragments over 2,966 peptides from one Stellar run. + +**Out-of-fold, which is the comparison that counts:** + +| fold | C# MAD (Th) | Python MAD (Th) | +|---|---|---| +| 1 | 0.0452 | 0.0451 | +| 2 | 0.0440 | 0.0440 | +| 3 | 0.0440 | 0.0440 | +| 4 | 0.0448 | 0.0448 | +| 5 | 0.0445 | 0.0446 | +| **pooled** | **0.0445** | **0.0445** | + +Same rows, same held-out peptides, same answer to four decimal places. An earlier +in-sample comparison put C# marginally ahead; cross-validation shows that gap was not real. + +**In-sample**, both trained on everything, which answers a different question - whether the +two learn the same *function* rather than merely reach the same accuracy: + +| | value | +|---|---| +| Pearson r between the two predictions | 0.9955 | +| Median absolute difference | 0.0034 Th | +| RMS difference | 0.0079 Th, which is 6.6% of the uncorrected spread | +| Max absolute difference | 0.127 Th | + +| Residual after correction | std | MAD | +|---|---|---| +| uncorrected | 0.1183 | 0.0802 | +| C# (`Osprey.ML`) | **0.0839** | **0.0431** | +| Python (XGBoost) | 0.0843 | 0.0433 | + +Two independent boosting implementations will never agree tree for tree, and per-peak +corrected m/z values do differ. What matters is that they learn the same function and leave +the same amount of error behind, and they do. + +Reproduce it with: + +```bash +mars calibrate --mzml run.mzML --prism-csv report.csv --no-dedupe-library \ + --validation-split 0 --no-recalibrate --dump-predictions cs.csv --output-dir out/ +python dotnet/scripts/compare_models.py --csharp cs.csv +``` + +The dump carries a `peptide_group` column, and `compare_models.py` reproduces MARS's fold +assignment from it - sort the distinct peptides, deal round-robin - so both sides train on +exactly the same rows and score exactly the same held-out peptides, rather than merely +similar ones. Pass `--cv-folds 0` to skip the Python-side cross-validation and compare +in-sample only. diff --git a/docs/mzml-passthrough.md b/docs/mzml-passthrough.md new file mode 100644 index 0000000..465febb --- /dev/null +++ b/docs/mzml-passthrough.md @@ -0,0 +1,162 @@ +# How MARS writes mzML + +MARS modifies one thing in a file: the m/z array of the MS2 spectra it corrects. Everything +else has to survive untouched, and that turns out to be harder than it sounds. + +- [Why passthrough](#why-passthrough) +- [The contract](#the-contract) +- [How it works](#how-it-works) +- [Binary arrays](#binary-arrays) +- [Index and checksum](#index-and-checksum) +- [Verifying output](#verifying-output) +- [Memory and speed](#memory-and-speed) + +## Why passthrough + +The obvious way to modify an mzML is to parse it into a document tree, change what you want, +and serialize it back. The Python implementation of MARS tried that twice - once with +[psims](https://github.com/mobiusklein/psims) and once with an lxml round trip - and both +produced files that were **valid mzML and still broke DIA-NN and SeeMS**. + +The reason is that a serializer is entitled to make changes a schema validator will not +object to: reordering attributes, normalizing whitespace, moving or re-declaring namespaces, +rewriting numeric formats. Downstream tools that parse mzML with hand-rolled scanners rather +than a full XML stack notice. + +So MARS does not serialize. It copies the input byte for byte and splices replacement bytes +into the specific ranges it is changing. Everything outside those ranges is identical to the +input by construction, not by care. + +## The contract + +Non-negotiable, all of it load-bearing for downstream compatibility: + +1. **Write indexed mzML.** DIA-NN fails silently on unindexed files. +2. **Preserve `cvRef="MS"`.** Never emit `cvRef="PSI-MS"`. +3. **Preserve the Thermo nativeID format** (`controllerType=0 controllerNumber=1 scan=NNNN`, + `MS:1000768`) and every source file reference. +4. **Re-encode a modified array with the same compression and precision it was decoded + with.** Read encoding per ARRAY, never per spectrum: m/z is typically 64-bit while + intensity is often 32-bit, and compression can differ between two arrays in one spectrum. +5. **Update `encodedLength`** on every modified array, to the base64 **character** count. +6. **Regenerate `indexList`, `indexListOffset` and the SHA-1 `fileChecksum`** after any + modification. +7. **Do not add or remove spectra**, and do not recompute derived CV terms (base peak m/z, + TIC) unless the correction actually invalidates them. + +## How it works + +The file is walked as a sequence of regions: + +``` +[gap: header, run metadata, spectrumList open tag] -> copied verbatim +[spectrum] -> parsed; m/z spliced if corrected +[gap: whitespace] -> copied verbatim +[spectrum] -> ... +... +[gap: chromatogramList, closing tags] -> copied verbatim +[chromatogram] -> copied verbatim, offset recorded +[trailer: index, indexListOffset, fileChecksum] -> regenerated +``` + +Spectrum and chromatogram elements are located by scanning for their start tags. Within a +spectrum, metadata is parsed with `XmlReader` over just that span - a real XML parser, so +attribute quoting and entity escaping are handled correctly - while the byte ranges to +splice are found by scanning. That split matters: `XmlReader` reports line and character +positions, not byte offsets, and the writer needs bytes. + +Metadata is read by **CV accession**, never by name. Names are display strings that vary +between writers, and pwiz emits a `` inside every +isolation window that a name-matching reader would happily mistake for the real +`MS:1000511`. + +A corrected spectrum is rebuilt as: + +``` +[bytes before encodedLength value] [new length] [bytes up to text] [new base64] [rest] +``` + +Two ranges change. Everything else in the element - every cvParam, the scan list, the +precursor list, the intensity array, the indentation - is the input's own bytes. + +## Binary arrays + +Decoding is base64 then, if declared, zlib. mzML uses the **zlib container**, with its +2-byte header and Adler-32 trailer, so `ZLibStream` rather than `DeflateStream`. Binary +arrays are little-endian by specification. + +Two details that cost real debugging time: + +**Inflate must not write into its own source buffer.** Reading compressed bytes from a +stream that wraps the destination array corrupts data that has not been consumed yet. It +is invisible on small spectra, because the whole payload fits in one internal buffered read, +and only appears once an array exceeds about 8 KB compressed. There is a regression test +that decodes a 20,000-peak spectrum specifically to keep that path honest. + +**Compressed bytes are not portable.** The zlib each runtime ships is not the same, so the +same input compresses to different bytes on Windows and Linux even though the values are +identical. Equivalence is therefore defined on decoded values throughout. + +## Index and checksum + +Both are regenerated from the bytes actually written, rather than adjusted from the input. +Offsets are recorded as each element is emitted, so they are correct by construction. + +The `fileChecksum` is SHA-1 over every byte from the start of the file **up to and including +the `` opening tag**. This was established empirically rather than from the +specification text: a pwiz-written file reproduces its recorded digest only under that +convention. + +> The Python implementation of MARS stops the hash two bytes earlier, before the indentation +> preceding ``. Every mzML it has written therefore carries a checksum that +> fails validation. Most consumers never check, which is why it went unnoticed. The C# +> writer uses the inclusive convention, and `mars verify` checks it. + +A plain `` file with no `` wrapper has nowhere to put an index, so MARS +copies it through unindexed and warns. Convert with msconvert first if DIA-NN is the +destination. + +## Verifying output + +The passthrough is testable independently of any science, which is the point: + +```bash +mars verify run.mzML +``` + +This applies a **null correction** - decode and re-encode every m/z array without changing a +value - and then checks that the result: + +- decodes to bit-identical m/z and intensity arrays, +- has an index whose every offset lands on the element it names, +- has a SHA-1 checksum that validates. + +If something looks wrong with a corrected file, run this first. It separates "the file +format handling is broken" from "the model is doing something strange", and those have very +different fixes. + +On the reference 1.2 GB Stellar file: 114,635 spectra, 56,972,925 peaks, m/z and intensity +bit-identical, index and checksum valid. + +To compare two files that were produced independently, use decoded values rather than `cmp`: + +```bash +mars compare a.mzML b.mzML --validate +``` + +## Memory and speed + +Never load the file. Memory is bounded by the largest single spectrum plus the training +matrix, so a 4.9 GB Astral run streams in the same working set as a 1.2 GB Stellar one. + +Pass 2 runs the per-spectrum decode, predict and re-encode across workers while writing in +order, so output byte order is unaffected by thread count. Inference carries no cross-row +accumulation, so parallelizing it cannot change a value. + +Measured on 16 cores: + +| | | +|---|---| +| Null-correction round trip, 1.2 GB | 6.9 s (176 MB/s) | +| Correct and write, per 1.2 GB file | 24 to 38 s | +| Full `calibrate` over 5 files, 6.0 GB in | 229 s | diff --git a/docs/open-questions.md b/docs/open-questions.md new file mode 100644 index 0000000..24e341e --- /dev/null +++ b/docs/open-questions.md @@ -0,0 +1,468 @@ +# Open questions + +Things deliberately left undone, with enough context to pick up cold. Each says what was +measured, what is unresolved, and what would settle it. + +## Generalizing a model across datasets + +**The goal:** train once on a well-characterized run and apply that model broadly to new +data from the same instrument platform, rather than fitting a fresh model per cohort. + +This is the most valuable open question here, and MARS is already built for it - `mars apply` +exists, the model file carries its feature list and acquisition-time offset, and +cross-validation already reports the number that matters for it. What has never been done is +the experiment. + +What is known so far: + +- Cross-validation estimates within-cohort generalization: a model scored on peptides it did + not train on. On the reference Stellar run that costs 0.0014 Th against a corrected error + of 0.0446 Th, about 3%. +- That is **not** the same question. Held-out peptides from the same run share its + instrument state, its acquisition window, its space-charge conditions. A different run does + not. +- The data-poor 400-500 window shows a 20% gap where the data-rich 600-700 window shows 3%, + so the estimate is sensitive to how much the fit had to work with. + +What would settle it, roughly in order: + +1. **Train on run A, apply to run B, re-match and measure.** The infrastructure is all + there: `mars calibrate --no-recalibrate` on A, `mars apply --model` to B, then `mars qc` on + the corrected B against the same library. Compare against fitting B directly. +2. **Vary the gap between A and B**: same plate, same day, different day, different column, + different instrument of the same model. The interesting output is where transfer stops + working, not whether it works at all. +3. **Watch `absolute_time` specifically.** It is re-based per fit, so a model carries A's + time origin. Transfer either has to re-base against B or drop the feature. This is the + most likely thing to break quietly. +4. **Consider what "same platform" means for the feature set.** A model trained with RF + temperature features cannot be applied to a run without those logs; loading fails cleanly + rather than silently, which is correct, but it constrains what a broadly applicable model + can use. + +If transfer works, the practical payoff is large: no library required for routine runs, and +a correction that does not depend on how many peptides happened to be identified. + +## Whether a robust loss beats trimming on more data + +`--robust trim` is the default and `--robust huber` is available. The measurement behind that +choice is thinner than it first looked: + +| | 600-700 window | 400-500 window | +|---|---|---| +| `trim` | 0.0442 Th | 0.0505 Th | +| `huber` | 0.0443 Th | 0.0518 Th | +| *fold-to-fold spread* | *0.0005 Th* | *0.0021 Th* | + +On the larger window the difference is one part in four thousand - noise. On the smaller one +trim is ahead, but by less than the spread between folds. Trim is the safer default, not a +demonstrated winner. + +The mechanism suggests Huber should do relatively better with more data: it errs by leaving a +mislabelled row about 79% of its weight, which costs most when there is little real signal to +outvote it. Worth re-running on a full cohort, and on Astral data, before treating the +default as settled. The Astral run is a particularly good test of this, at 1.4 million rows +over 81 thousand peptides - an order of magnitude more of both than the Stellar windows the +current default was chosen on. See [model.md](model.md#why-trimming-rather-than-a-robust-loss). + +## A redescending weight + +The untried middle between trimming and Huber. Tukey's biweight, +`w = (1 - (r/c)^2)^2` for `|r| <= c` and `0` beyond, goes to exactly zero past a multiple of +the threshold instead of decaying as `1/|r|`. That would soften the boundary - no cliff for a +row to sit astride - while still eliminating the far tail, which is the property trimming has +and Huber lacks. + +About ten lines in `MzCalibrator.TrainRobust`, plus a decision about how `--robust-sigma` +should scale for it: Tukey down-weights *within* the threshold too, so the conventional +constant is around 4.685 sigma rather than 3. + +## A native Huber objective in Osprey.ML + +The current `--robust huber` reaches Huber by reweighting and refitting, which is exact for +the gradient but applies the robustness once rather than at every boosting round. A real +objective upstream would re-clip each round as the residuals shrink, and would avoid the +second full pass. + +Only worth doing if the reweighted version proves valuable first. It is a pwiz change with +the same bit-identity discipline PR #4595 established, and should follow that PR rather than +stack on it. + +## Hyperparameter tuning + +Settled for now: **do not**. A sweep of 13 configurations from 50 trees at depth 4 to 400 at +depth 8 leaves out-of-fold MAD flat at 0.0445-0.0449 while in-sample drops from 0.0446 to +0.0347, and the largest configuration is the worst out-of-fold. See +[model.md](model.md#hyperparameters). + +Worth revisiting only if the error floor moves - if the mismatched-peak population is dealt +with better, or if a transferred model turns out to be capacity-limited rather than +noise-limited. + +## The Python CLI's import cost + +`mars/cli.py` imports `__version__` from the package root, which runs +`mars/__init__.py` and so eagerly imports the submodules and their dependencies on +every invocation - including `mars --version`. Reading the distribution metadata +directly would avoid both the side effects and the startup cost. + +Raised by the Copilot review on PR #9 and **not applied**: the Python implementation is +frozen to bug fixes, and startup cost is not a bug. Worth doing only if that track is +unfrozen; if it is retired as planned, this closes with it. + +## Reading vendor RAW directly, via pwiz-sharp + +**Measured, decision pending.** [ProteoWizard PR #4178](https://github.com/ProteoWizard/pwiz/pull/4178) +ports the ProteoWizard core to .NET 8, including the Thermo `.raw` reader and the mzML +writer. If MARS used it, the workflow would go from `RAW -> msconvert -> mzML -> MARS` to +`RAW -> MARS`. + +### What was tried + +A shallow sparse clone of `chambem2/pwiz-sharp` (44 MB), building +`pwiz/src/Vendor/Thermo/Thermo.csproj` with `-p:IAgreeToVendorLicenses=true`, then a throwaway +probe against a 4.9 GB Astral run (`Ast_20240220_S10_26.raw`, 121,290 spectra). + +**It works, and it gives MARS everything it needs.** Every field the matcher reads is present +on the pwiz `Spectrum`: ms level, scan start time, ion injection time, isolation window target +with lower and upper offsets, total ion current, the Thermo filter string, and the m/z and +intensity arrays. Injection time and isolation window were present on 120,327 of 120,327 MS2 +spectra. Opening the file costs 1.6 s, because the reader is lazy and only the header is read. + +The run declares **two instrument configurations** - `IC1` quadrupole + orbitrap, `IC2` +quadrupole + Astral analyzer - which is the same hybrid layout the mzML analyzer detection +handles, reached the same way. Detection would carry over to RAW input unchanged. + +### What argues against it + +**Reading RAW is not faster, and does not thread.** Full read with binary data: + +| Threads | Wall | Throughput | +|---:|---:|---:| +| 1 | 85.8 s | 3.38 M peaks/s | +| 2 | 108.5 s | 2.67 M peaks/s | +| 4 | 72.5 s | 4.00 M peaks/s | +| 8 | 72.5 s | 4.00 M peaks/s | +| 12 | 72.9 s | 3.98 M peaks/s | + +Flat from four threads on, with one reader handle per worker and striped indices. For scale, +MARS's whole match pass over a comparable Astral mzML is 41 s - a different acquisition, so +not a like-for-like comparison, but enough to say RAW reading is not the faster path. The win +would be removing the conversion and its ~5 GB intermediate, not the read itself. + +**A Thermo-only build drags a native Windows DLL.** `Thermo.csproj` references +`Analysis.csproj`, which references `Waters.csproj`, which stages `MassLynxRaw.dll` - a +Windows x86-64 native PE - into the output. Nothing in the Thermo reader touches Waters, so +this looks vestigial upstream, but MARS ships `linux-x64`, `linux-arm64`, `osx-arm64` and +`osx-x64` and would be carrying it. The managed Thermo SDK itself is cross-platform; this is a +project-reference shape, and worth raising upstream rather than working around. + +**It is not consumable as a package.** No `GeneratePackageOnBuild`, so there is no NuGet +artifact; MARS would vendor or submodule a build of an unmerged draft branch. The tree is also +not self-contained - `Common.csproj` embeds `pwiz/data/common/{psi-ms,unimod,unit}.obo` from +the C++ tree, and the build needs `libraries/7za.exe` - so a sparse checkout has to include +those paths. The vendor SDK is license-gated behind `-p:IAgreeToVendorLicenses=true`, which +MARS's CI and release build would have to carry deliberately. + +**The port is a draft at 85% semantic parity** with C++ msconvert (359 of 421 comparable +files identical), with the remaining differences documented as mostly not port defects. + +### The writer, measured + +MARS's output guarantee is the [byte-splice passthrough](mzml-passthrough.md), and reading +RAW removes the input that splice is made against. The concern that motivated the splice was +serializer damage: two round-trips in the Python implementation produced valid mzML that broke +DIA-NN and SeeMS. That concern does **not** transfer to pwiz. Those round-trips were psims and +lxml; the mzML MARS reads was written by msconvert, so writing with pwiz-sharp regenerates a +file with the same lineage rather than introducing a foreign serializer. Handing the format +code to the people who maintain the format is the point. + +Measured, on `Ste-2024-12-02_HeLa_20msIIT_GPFDIA_600-700_16`, applying the same model through +a `SpectrumListWrapper` and writing with `MSDataFile.Write`, then diffing against MARS's own +byte-splice output with `mars compare`: + +``` +spectra compared 114,021 +peaks compared 82,349,582 +m/z values differing 0 +max |delta m/z| 0 Th +intensity differing 0 +``` + +**Numerically identical.** That settles the two things worth settling: the adapter from pwiz's +`Spectrum` to MARS's `SpectrumRecord` feeds the model the same values the native reader does, +and the writer round-trips them without loss. + +### What the writer costs + +| Output | Size | Wall | Input | +|---|---:|---:|---| +| mzML (byte-splice, for reference) | 1.906 GB | - | 1.471 GB | +| mzML (pwiz) | 1.916 GB | 337 s | 1.471 GB | +| mzXML (pwiz) | 0.983 GB | 226 s | 1.216 GB | +| mzMLb (pwiz) | 0.557 GB | 213 s | 1.216 GB | + +Both mzML writers inflate relative to the input, which is expected: correcting m/z makes the +arrays less compressible than the smooth originals. pwiz lands within 0.56% of the splice. +mzMLb is less than half the input, which is an argument for it on its own. + +Two things to get right: + +**Match the encoding explicitly.** `BinaryEncoderConfig` defaults to 64-bit *uncompressed*, +which inflated the first attempt by 61%. The input is 64-bit zlib for both arrays, and setting +that recovers the size. Note the shape difference: MARS's splice reads encoding **per array**, +because m/z is often 64-bit where intensity is 32-bit and compression can differ between two +arrays of one spectrum. pwiz's config is global, with per-array overrides keyed by CVID - so +the common case is expressible, but a file whose encoding varies spectrum to spectrum is not. + +**The write path is sequential and that is the real cost.** MARS's byte-splice writer is +parallel and writes 8.4 GB across five files in 123 s; the pwiz spike reads, corrects and +writes one file in 337 s single-threaded. pwiz's `ISpectrumList` is random-access so the work +parallelizes in principle, but `MSDataFile.Write` pulls spectra sequentially. Closing that gap +is the main engineering cost of the move, and it is a throughput problem rather than a +correctness one. + +### Decision + +**Adopt the pwiz writer.** It buys mzXML and mzMLb now (mz5 has no writer class yet; mzMLb has +one and is dispatched from the path-shaped `Write` overload, not the stream-shaped one), it +puts the format code with the people who maintain the format, and it produces byte-for-byte +equivalent numbers on real data. + +Staging, smallest dependency first: + +1. **Writer only, mzML in.** Needs `Util`, `Common` and `MsData` - no vendor projects, so no + native Waters DLL. `SpectrumListWrapper` lives in `Analysis`, so either take that reference + or derive from `SpectrumListBase` in `MsData` to keep the dependency to three projects. +2. **Parallelize the write**, to close the throughput gap against the splice. +3. **RAW input**, which adds the vendor chain and its Windows-only transitive DLL. + +Keep the byte-splice path for mzML in and mzML out until 2 lands, then decide whether to +retire it on the evidence rather than in advance. Note that mzMLb output on `linux-arm64` and +`osx-arm64` needs checking: `HDF.PInvoke.1.10` bundles native libhdf5 for Windows and Linux +**x64**, and MARS ships arm64 artifacts. + +Reproduce with the probes under `scratchpad/` - `rawprobe` for the reader, `pwizwrite` for the +wrapper and writer. + +## Ion mobility: collapsed, not modelled + +**Settled.** MARS does not read ion mobility and does not intend to. It collapses the dimension +instead, by asking pwiz to combine each TIMS frame's mobility scans back into one spectrum per +isolation window. + +The reason it came up: pwiz presents an uncombined TIMS frame as hundreds of spectra sharing +one retention time and one isolation m/z, separated only by mobility. ProteoWizard's +`diaPASEF.d` is 4,631 spectra at five distinct scan times, and the first MS2 in it holds +**two peaks**. Combining turns the same file into 8 MS2 spectra across 8 distinct isolation +windows and 4 retention times, and that first spectrum into **8,377 peaks**. + +That is not only tidier, it is the difference between usable and not. MARS matches library +fragments within a spectrum and computes its space-charge features from the peaks around each +match - `ions_above_0_1`, `adjacent_ratio_1_2` and the rest. On a two-peak mobility slice there +are no neighbours to measure, so those twelve features would be noise even where they were +defined. Combined, the spectrum has the shape every other instrument already produces, and the +matcher and the features work unchanged. + +Reading and writing both combine, so what gets written matches what was matched and modelled. + +The alternative - carrying mobility as a feature and fitting the uncombined slices - was +tried and reverted. It would have meant a 23rd feature that only one vendor populates, and a +matcher operating on spectra too sparse to compute most of the others from. + +## Where the vendor SDKs come from + +**Decided for now: MARS ships them.** A released binary carries the Thermo, Bruker and Sciex +assemblies, so a download opens a `.raw` with nothing else installed. That is what pwiz and +Skyline already do, and it is the only option that makes "download MARS, calibrate a run" true. + +**Planned to change once [PR #4178](https://github.com/ProteoWizard/pwiz/pull/4178) merges to +master.** When Skyline and msconvert ship pwiz-sharp themselves, MARS should stop carrying its +own copies and use the installed ones - the user has already accepted the vendor licences by +installing either. + +Three candidates, in preference order, and the order is about how stale each one's SDK is +likely to be: + +| Candidate | Updates | Discovery | +|---|---|---| +| **Skyline-daily** | ClickOnce, frequent | Verified below; both channels verified | +| **Skyline** | ClickOnce, much less often | Same mechanism and the **same** token; see below | +| **msconvert** | Manual download only | Unreliable - see below | + +Skyline-daily first because it updates fastest, so its SDK will track upstream without MARS +releasing anything. Regular Skyline uses the same ClickOnce machinery and should be used when +it is the only one present. + +**Both channels ship the same Thermo SDK today**: 5.0.0.93, on Skyline 26.1.0.57 and +Skyline-daily 26.1.1.209 alike, both installed here. So the ordering is about which will move +first once pwiz-sharp merges, not about a difference that exists now. msconvert last: it does +not update itself at all, and its registry entry is the least useful of the three. + +**Because they lag by different amounts, the version has to be checked rather than assumed.** +Whatever is found, read the `FileVersion` of `ThermoFisher.CommonCore.RawFileReader.dll` and +compare it against the minimum pwiz-sharp needs before using it; fall through to the next +candidate, and finally to a bundled copy, when it is too old. That turns "prefer daily" from a +guess into a check - and it is the same check that catches today's 5.0.0.93 immediately rather +than at the first `MissingMethodException`. + +### Why it cannot be done today + +Tried, and it fails for a version reason rather than a licensing one. Skyline is installed on +the development machine and does ship `ThermoFisher.CommonCore.RawFileReader.dll` - at +**5.0.0.93, targeting .NET Framework 4.7.1**. pwiz-sharp needs **8.0.6.0**: a `net8.0` process +cannot load the former, and pwiz-sharp calls `RawFileReaderAdapter.ThreadedFileFactory` and the +three-argument `Scan.FromFile`, which only the newer SDK has. The C++ ProteoWizard install +carries the same 5.0.0.93. There is no NuGet package. + +So the precondition is not "Skyline is installed" but "Skyline ships pwiz-sharp's assemblies", +which is what #4178 merging brings. + +### What the switch will need + +**Finding the install needs a registry lookup, and the registry does not hold the path.** +ClickOnce puts Skyline under a hashed directory that changes on every update, so a directory +constant is not an option. The route from registry to files, confirmed against the +Skyline-daily installed on the development machine: + +1. Enumerate `HKCU:\Software\Microsoft\Windows\CurrentVersion\Uninstall`. The subkey names are + opaque hashes - `6e917b9fd968e06d` here - so match on `DisplayName`, which is `Skyline-daily` + (a second entry, `Skyline-daily Parquet`, is a tool rather than the application). + +2. The entry has **no `InstallLocation`**. What it does have is the ClickOnce identity, in + `UninstallString` and `ShortcutAppId`: + + ``` + rundll32.exe dfshim.dll,ShArpMaintain Skyline-daily.application, Culture=neutral, + PublicKeyToken=9286511f3362df93, processorArchitecture=msil + ``` + + Take `PublicKeyToken`. Keep `DisplayVersion` too - `26.1.1.209` here - it disambiguates in + the next step. + +3. That token is embedded in the deployment directory name under + `%LOCALAPPDATA%\Apps\2.0`: + + ``` + ...\Apps\2.0\\\skyl..tion_9286511f3362df93_001a.0001_6c454ec13578dbec\ + ``` + + **The token does not identify which Skyline.** Skyline and Skyline-daily are signed with + the same key and share `9286511f3362df93`, so both match the same glob - an earlier + version of this note assumed they differed, and an implementation built on that would + have picked whichever it enumerated first. Several directories match for the further + reason that previous versions are kept, and `..exe_` folders sit beside `..tion_` ones + carrying no vendor DLLs at all. + + Identify the right one by the **executable it contains** - `Skyline.exe` against + `Skyline-daily.exe` - and confirm with the version. Compare versions **parsed, not as + strings**: the registry says `26.1.0.57` where the file says `26.1.0.057`, equal as a + version and unequal as text. Do not select by newest timestamp; an update in progress + would make that lie. + +That directory is the one holding the vendor assemblies; the `..exe_` folders hold none, which +is a cheap way to reject them. On this machine both channels hold +`ThermoFisher.CommonCore.RawFileReader.dll` at **5.0.0.93** - Skyline 26.1.0.57 and +Skyline-daily 26.1.1.209 - which is the version gap above, measured on current builds of +both rather than assumed. + +**msconvert is the awkward one to find.** ProteoWizard is installed on the development +machine and registers under `HKLM`, but the entry carries **no `InstallLocation`**, there is no +`App Paths` entry for `msconvert.exe`, and it is not under `Program Files`. Its +`UninstallString` is a bare `MsiExec.exe /I{GUID}`, which names the product without saying +where it went. Resolving it would mean querying the Windows Installer product database rather +than reading a value. That is a second, independent reason to reach for it last. + +macOS and Linux need something else entirely - Skyline is Windows-only - so those platforms +keep the bundled SDKs regardless. The switch is a Windows optimisation rather than a change of +approach everywhere, which means the bundling machinery stays either way. + +**Only the vendor SDKs move.** mzMLb needs a native HDF5 through `HDF.PInvoke`, which is not a +vendor library and would stay bundled. + +**`mars --version` already reports what a binary carries**, and should keep telling the truth +across the change: a MARS that borrows its readers from Skyline and cannot find one has to say +so there, rather than at the moment somebody opens a file. + +## The ZenoTOF 8600 reads, but reports no analyzer + +**Measured on real data, and it needs an upstream fix.** + +A ZenoTOF 8600 `.wiff2` opens and reads correctly: 300,570 spectra, 3,000 MS2 sampled with +11.7 million peaks across 429 distinct isolation windows, isolation window present on every +one. The bundled Sciex SDK handles the instrument. + +What it does not do is say what recorded it. pwiz's Sciex model table stops at `ZENOTOF7600`, +so an 8600 falls off the end: + +``` +[Reader_Sciex.FillInMetadata] unable to determine instrument model + (unknown instrument type: ZenoTOF 8600 System) +``` + +That is non-fatal by design, but the consequence is not cosmetic. With the model unrecognised +the reader emits an instrument configuration with **no components at all** - `IC1: (no +components)` - and Sciex writes no Thermo-style filter string. MARS therefore has nothing to +classify from, reports `Unknown`, and falls back to the unit-resolution default of 0.3 Th. +On a TOF that is about **760 ppm at m/z 400**, against real error of a few ppm. + +### The upstream fix + +One line in `pwiz/src/Vendor/Sciex/Reader_Sciex_Detail.cs`, alongside the existing entry: + +```csharp +if (n.Contains("ZENOTOF7600", StringComparison.Ordinal)) return SciexInstrumentModel.ZenoTOF7600; +``` + +An 8600 case, and a `SciexInstrumentModel` member mapping to `MS_time_of_flight` the way +`ZenoTOF7600` does. Worth raising on +[PR #4178](https://github.com/ProteoWizard/pwiz/pull/4178), because every tool reading 8600 +data through pwiz-sharp inherits this, not only MARS. + +### What MARS does about it meanwhile + +Detection cannot be fixed from MARS's side - there is genuinely no analyzer information in the +file as pwiz presents it. So MARS checks the consequence instead, after matching, when the data +can answer: if the matching window is more than 50x the median absolute error actually found, +it says so. A window that wide is the signature of a tolerance set for the wrong instrument. + +The asymmetry is what makes this worth doing. A tolerance that is too narrow fails loudly, with +too few matches to train on. One that is too wide fails silently - it fills with peaks that are +not the fragment, and the run completes and reports numbers regardless. Only the silent +direction needs a detector. + +The threshold is loose deliberately: trap data at its correct 0.3 Th sits around 4x, so this +cannot fire on the case MARS was built for. Verified both ways - silent on a Stellar run at +0.3 Th, and firing on high-resolution data forced to the trap tolerance. + +## Profile data is centroided by the vendor + +**Settled.** Sciex writes profile spectra, and MARS asks the vendor to centroid them before +matching or correcting anything. + +The ZenoTOF 8600 file is stored as profile: `profile=True`, 1,619 points in one MS2, evenly +spaced at 0.00233 Th. That spacing is 16 ppm at m/z 142, which matters because MARS measures +mass error by taking the most intense peak inside a tolerance window - on a sampled curve the +answer is quantised to the grid, so the floor on measurable error would be several times the +error the instrument actually has. The twelve space-charge features fare worse still: they +count the peaks around a match, and on profile they would count samples of the same ion. + +pwiz exposes the vendor's own algorithm through `IVendorCentroidingSpectrumList`, which +`SpectrumList_Sciex` implements as "ABI/Analyst peak picking". The vendor knows its detector, +so that is preferred over a peak picker of ours. On the first MS2 of that file it turns 1,619 +points into **210**, and across 200 MS2 the average falls from about 3,900 samples to 801 +peaks. + +Applied on **both** paths, reading and writing. They have to agree: the model is fitted on peak +lists, and correcting sampled curves with it would put every feature far outside anything it +saw in training. + +**Only when the spectrum says it is profile.** Thermo and Bruker already deliver centroids - +an Astral run reads `centroid=True`, 139 peaks, and passes through untouched - so this changes +nothing for them. + +One consequence worth being explicit about: a corrected file written from profile input comes +out **centroided**, because that is what was modelled. That is what `msconvert --filter +peakPicking` does routinely and what DIA-NN and Skyline want, but it is a real change to the +data rather than only to the m/z values, and it is the one case where MARS's output differs +from its input in more than the numbers it set out to correct. diff --git a/docs/python-parity.md b/docs/python-parity.md new file mode 100644 index 0000000..cd5d310 --- /dev/null +++ b/docs/python-parity.md @@ -0,0 +1,108 @@ +# Checking the C# implementation against the Python one + +The Python implementation is the reference. It has been used on real data for long enough +that "the C# version computes the same thing" is a stronger statement about correctness +than any test written from first principles, because a test can only check what its author +already believed. + +This page describes how that comparison is made and what it does and does not cover. + +## Why aggregate agreement is not enough + +The two implementations agree on the numbers that appear in a QC report. On the five-file +Stellar cohort both find **352,349 matches**, and measured on the written files: + +| | before | C# | Python | +|---|---|---|---| +| MAD | 0.0800 Th | 0.0464 | 0.0472 | +| std | 0.1180 Th | 0.0872 | 0.0882 | + +Both columns come from one paired run, re-taken after the injection-time fix moved the C# +column. Python landed within 0.0001 Th of its previous run, which is what says the two +measurements are comparable. Both implementations use the same 20 features on this cohort. + +That is reassuring and almost meaningless on its own. It is a summary over hundreds of +thousands of rows, and a feature can be wrong in a way that never moves it. The two +features the model weights most heavily - `ions_above_0_1` at 0.346 importance and +`adjacent_ratio_0_1` at 0.292 - are sums over neighbouring peaks. An off-by-one in the +window boundary changes every row a little, the model re-fits around it, and the corrected +spread barely moves. + +So the comparison has to be per row and per feature. + +## Running it + +Both sides write the same CSV schema: one row per matched fragment, identified by the scan +and the library fragment, carrying every feature the model will see. + +```bash +# C# +mars calibrate --mzml run.mzML --prism-csv report.csv \ + --no-dedupe-library --no-recalibrate --dump-matches cs.csv --output-dir out/ + +# Python +python dotnet/scripts/dump_python_matches.py \ + --mzml run.mzML --prism-csv report.csv --out py.csv + +# Difference them +python dotnet/scripts/compare_matches.py --csharp cs.csv --python py.csv +``` + +`--no-dedupe-library` is required. The C# reader collapses transitions that repeat across +replicates and the Python one does not, so without it the two produce different row sets +for a reason that has nothing to do with correctness. + +`compare_matches.py` exits non-zero on any disagreement, so it can gate a build. + +### What the comparison does + +- **Joins on (scan number, ion annotation, expected m/z).** Peptide sequence is + deliberately excluded: each side carries whatever form its library reader produced, and a + formatting difference there is not a calibration difference. +- **Pairs repeated keys by position** rather than discarding them. A few precursors appear + in more than one block of a PRISM report, so the same fragment can be matched twice in + one scan through two library entries. Both implementations produce those duplicates; each + group is ordered identically on both sides and paired off. A difference in the multiset + would surface as an unmatched row. +- **Treats NaN on both sides as agreement**, and NaN on one side as a failure. NaN is how an + undefined ratio reaches row selection, so the two agreeing that a row is undefined is + agreement about the row's fate. +- **Reads with `float_precision="round_trip"`.** Pandas' default float parser drops the last + digit, which invents differences of about 1e-16 in every column and buries the real ones. + +## Result + +On `Ste-2024-12-02_HeLa_20msIIT_GPFDIA_400-500_14.mzML` with the Stellar PRISM report: + +``` +Row counts C# 14,432 Python 14,432 +Row agreement matched by both 14,432 C# only 0 Python only 0 + +24 columns compared, max absolute difference 0.000e+00 on every one. +``` + +Every feature, every row, bit-identical - including all six space-charge features and the +six ratios derived from them, `absolute_time` after re-basing, `log_tic`, `log_intensity`, +`injection_time` and `tic_injection_time`. + +The comparison was checked against a perturbed copy to confirm it can fail: a 1e-8 shift in +one `observed_mz`, a 1% shift in one `ions_above_0_1`, and one ratio forced to NaN were all +detected and reported. + +## What this does not cover + +Parity is the right standard for the parts that were transcribed. It is not the standard +for everything: + +- **The four deliberate divergences.** MARS in C# does not reproduce four defects in the + Python implementation; see [the port spec](dotnet-port-spec.md) section 10a. Two of them + change training and can be reproduced with `--python-compat` for an A/B run. +- **The model itself.** The gradient boosted trees are a different implementation, so + per-peak corrected m/z values differ. Agreement there is statistical, not exact. +- **Everything Python has no counterpart for**: `mars verify`, the byte-splicing writer, + the `--on-reorder` policies, command-line parsing, and the cross-platform packaging. +- **The `.blib` and DIA-NN paths**, which have their own readers on both sides. The same + harness applies; only the PRISM path has been run through it so far. + +For those, ordinary tests are the only option, and the gaps are recorded in the test +coverage notes rather than papered over. diff --git a/docs/qc-report.md b/docs/qc-report.md new file mode 100644 index 0000000..eb83aff --- /dev/null +++ b/docs/qc-report.md @@ -0,0 +1,253 @@ +# Reading the QC report + +Both `mars qc` and `mars calibrate` write two reports: + +- **`mars_qc_summary.txt`** - the numbers, for a pipeline or a quick look. +- **`mars_qc_report.html`** - the same numbers plus the figures, as one self-contained file. + +Mass error is expressed in **Th on unit-resolution data and ppm on high-resolution data**, +following whichever analyzer MARS read out of the mzML. Both scales appear in the summary +tables regardless; see [Th and ppm](#th-and-ppm). + +`qc` runs before any model exists, so its report shows the error **as measured** and how it +varies with each feature. `calibrate` shows the same figures with an after-correction +overlay, plus an after-heatmap, feature importance and cross-validation. Everything below applies to both; +where they differ it says so. + +The HTML file has no scripts, no external references, and fetches nothing when opened. +Everything is embedded, so it can be attached to an email and read by someone who has +neither the data nor the tool. A 22-feature report is around 550 KB. + +`--no-html-report` skips it; `--html-report ` moves it. + +## Th and ppm + +Every summary figure is reported on both scales, side by side: + +``` + Mean delta: -0.0005 Th -0.85 ppm + Median delta: -0.0009 Th -1.53 ppm + Std delta: 0.0029 Th 4.62 ppm + MAD delta: 0.0014 Th 2.27 ppm + RMS delta: 0.0029 Th 4.68 ppm + MAE delta: 0.0022 Th 3.55 ppm +``` + +Both, because neither one is sufficient on its own. A trap is specified in Th and its error +really is roughly constant in Th, so Th is the scale on which its performance is flat and +comparable across the m/z range. A high-resolution analyzer is specified in ppm and its +error is roughly constant in ppm, so on that instrument a Th figure is the one that drifts +with m/z and cannot be compared to the vendor number. Reporting only one scale makes MARS +look either wrong or arbitrary depending on which instrument wrote the file. + +The conversion is **per row, from each fragment's own m/z**, and then summarized - not the +aggregate Th figure divided by some nominal mass. The distinction is not cosmetic: fragments +in a plasma digest span most of a factor of four in m/z, so the shortcut would be off by +about that factor at the ends of the range. It also means the ppm and Th columns are not +rescalings of one another. Each is a summary of a different per-row quantity, and the ratio +between them varies with the m/z distribution of whatever matched. + +Which one to read is a property of the instrument, not of the report. On the Stellar data +below the interesting numbers are hundredths of a Th; on Astral data they are single-digit +ppm. + +## The verdict line + +At the top, before any figure. After `calibrate`: + +> **46.2% reduction** in median absolute error, 0.0802 → 0.0432 Th. The correction removed +> a substantial part of the mass error. + +After `qc`, where there is no model to report on: + +> **0.0802 Th** median absolute error across 146,515 matched fragments, with a median of +> -0.0042 Th. The median is close to zero, so there is no large constant offset. Whether the +> spread is systematic enough to remove is what fitting a model would show. + +The `qc` line deliberately stops short of predicting how much is removable, because nothing +short of fitting a model answers that. What it can say is how much of the error is a plain +constant offset - a median far from zero - which is the most straightforwardly correctable +thing there is. + +Median absolute deviation rather than standard deviation, because a handful of badly +matched peaks move a standard deviation and should not be allowed to decide whether the run +worked. + +**A small number here is a legitimate result, not a failure.** On an already +well-calibrated instrument there is little systematic error to remove, and the report says +so rather than implying something went wrong. If it reports that essentially nothing was +removed, the correct response is usually to keep the original files. + +## Mass error distribution + +The uncorrected error against what is left after correction, overlaid. After `qc` there is +only the one distribution: the error as measured. + +This is the headline figure and it is close to sufficient on its own. If the two +distributions are not visibly different, nothing further in the report matters. + +What to look for: + +- **A narrower after-distribution.** That is the whole point. +- **A shift toward zero.** The before-distribution is often offset - a systematic bias + across the whole run - and correcting that alone is worth a lot. +- **A remaining spread that is roughly symmetric.** What is left should look like noise. A + residual distribution that is still lopsided means there is structure the model did not + capture. + +The axis is bounded at a high percentile of the uncorrected error, so a few extreme rows +cannot squash the informative part into a sliver. + +## Error across retention time and fragment m/z + +Two panels after `calibrate`, before and after; one after `qc`. Color is the **median** +error in each cell - median, not mean, because a few mismatched peaks in a sparse cell +would otherwise invent structure that is not there. + +This is the most useful figure in a `qc` report. Visible structure means the error is +systematic, and systematic error is the kind MARS can remove; a featureless panel means it +is mostly noise, and calibrating will not achieve much. + +This is the figure that tells you the error is *systematic* rather than random, and +therefore correctable at all: + +- **Visible structure in the before panel** - bands, gradients, blocks - is systematic error. + That is what MARS removes. +- **A washed-out after panel** is the goal. Color surviving in the same places means the + model did not capture that region. +- **Structure along the m/z axis alone** is the classic mass-axis miscalibration. +- **Structure along the time axis** is drift during acquisition. +- **Blocky vertical bands** usually track the GPF or DIA isolation scheme rather than + anything physical, since each precursor window contributes a distinct set of fragments. + +Empty cells are simply where no fragment matched. + +## Cross-validation + +*`calibrate` only.* + +Per-fold accuracy, the pooled out-of-fold figure, and the spread across folds. Folds are +split by peptide, so every row was scored by a model that never saw its peptide. + +Read the **spread** row, and the two figures below the table that plot it. One held-out +number tells you how the model did on one split; five tell you whether that number was +luck. + +Each figure places every fold's value on an axis, marks the pooled figure, and shades one +standard deviation either side of the fold mean: + +- **Folds clustered together** - the estimate is stable, and the pooled number can be quoted + as-is. On the reference Stellar run the five folds span 0.0440 to 0.0452 Th, a standard + deviation of 0.0004 Th against a corrected error of 0.0446 Th, so which peptides happened + to land in which fold barely matters. +- **Folds scattered across the band** - the cohort contains regions the model handles very + differently, and the pooled figure is an average over them rather than a description of + any of them. Worth finding out what separates the good folds from the bad before trusting + the correction. +- **One fold well away from the rest** - usually a peptide population that behaves + differently: a different charge state, a different elution region, a contaminant set. + +Two metrics are plotted: median absolute residual, which is the accuracy, and Pearson r, +which is how much of the error's structure the model tracks. They can disagree, and it is +informative when they do - a fold with a good MAD but a poor r is one where there was little +error to find rather than one the model handled well. + +Then read the **gap**: the difference between what the correction leaves on the data it was +fitted to and what it leaves on peptides it never saw. + +This is not a measure of cheating. The correction model is fitted to all the data on +purpose - calibrating a run from species identified within it is what mass calibration is - +and the correction moves a peak onto a fitted surface rather than onto its theoretical m/z, +so there is little scope to memorize individual peaks. What a large gap does say is that the +surface is being driven by the particular peptides in this run rather than by the +instrument: the fit is thin, and `mars apply` would disappoint on other files. + +Both numbers appear at the top of the report: + +- **After Calibration (these files, corrected)** - what the corrected output will look like + when re-matched. This is the one to quote for the run in hand. +- **Expected on data not used to fit** - what the same procedure achieves on a run it was + not fitted to. This is the one to quote for `mars apply`. + +## Feature importance + +*`calibrate` only - there is no model to interrogate after `qc`.* + +Permutation importance: how much the validation error degrades when one feature's values +are shuffled, normalized to sum to 1. + +Split counts are not used, because a feature with many distinct values accumulates splits by +offering more places to cut whether or not those cuts help. Permutation importance measures +what the model would actually lose. + +A feature near zero is carrying no weight. On the reference Stellar cohort the two RF +temperature features score below 0.01 each - which is why the advice is that temperature +logs are worth using when you have them and not worth chasing when you do not. + +## Error against each feature + +One figure per active feature, before and after correction side by side. Color is the +**fragment count** per cell - dark purple through green to yellow - and the +line over it is the **median error per column**. + +The count scale is viridis rather than a single-hue ramp because a monochrome ramp has one +usable dimension and spends most of it on pale values, so the dense core and the sparse tail +end up looking alike. Counts map onto it as a power law (`count / peak` to the 0.4, as in +matplotlib's `PowerNorm`) rather than linearly or logarithmically. Linear leaves one bright +cell in a dark field, since the core of these densities runs orders of magnitude above the +tails; a log overcorrects, pushing most of the core to the yellow end so that the structure +inside it - the part worth looking at - washes out. Each panel is normalized to its own busiest cell: correcting +concentrates the distribution, so on a shared scale the before panel would flatten to nearly +empty and the structure that motivated the correction would vanish from the figure. Both +panels do share one vertical range, because the after panel being visibly tighter is the +result. + +Read the line first. The density shows where the fragments are, which is worth knowing - +it says which part of the axis the trend is actually supported by - but the median line is +the trend the model has to capture. + +- **A sloped or curved line in the left panel** is a real dependence, and the model should + be flattening it. +- **A flat line in the right panel** means the model captured that dependence. +- **A line that still slopes on the right** is error the model left behind. +- **A flat before-line** means that feature carries no information about the error here, and + should be near zero in the importance chart. + +After `qc` there is only the one line, and it is read as a forecast: a sloped line is a +dependence a model could exploit, and a panel full of flat lines is a warning that there +may be little for one to learn. + +A column with fewer than 20 rows is skipped rather than plotted, because a median over a +handful of points reads as signal when it is noise. + +The panels cover every feature that was available, so the set changes with the data: no +injection time in the file means no injection-time panels, and temperature panels appear +only when `--temperature-dir` was supplied. + +## How the figures are made + +Worth knowing because it explains the one visible artifact. + +Axes, labels, trend lines and bars are SVG written directly. The density layers are PNGs +embedded as data URIs, encoded by a small writer over the zlib already used for mzML. There +is no plotting dependency: every managed charting library for .NET either wraps a native +rasterizer or brings a large dependency tree, and avoiding exactly that is part of why the +C# implementation exists. + +Going straight to SVG rectangles produced a 6 MB file - one rectangle per cell, 76,000 of +them - which is not emailable. As quantized rasters the same figures come to 213 KB. + +The artifact: the density layers are raster, so they do not follow the reader's light or +dark theme the way the vector layers do. The ramps run light-to-dark, which reads correctly +on either. + +## What the report does not tell you + +- **Whether the library was right.** A model trained against "theoretical" m/z values that + carry someone else's calibration error will produce a confident-looking report and wrong + corrections. See [spectral libraries](spectral-libraries.md). +- **Whether the file is well-formed.** That is `mars verify`. +- **Whether the correction generalizes.** The validation MAE in the summary is the only + guard, and it is computed on rows from the same cohort. +- **How much of the error is removable**, in a `qc` report. Only fitting a model answers + that, which is what `calibrate` does. diff --git a/docs/spectral-libraries.md b/docs/spectral-libraries.md new file mode 100644 index 0000000..1c80fd4 --- /dev/null +++ b/docs/spectral-libraries.md @@ -0,0 +1,193 @@ +# Spectral libraries + +MARS needs a source of **theoretical** fragment m/z values. Everything downstream depends on +that word: the label is `observed - theoretical`, so if the "theoretical" side carries +measurement error of its own, the model learns to reproduce someone else's calibration +rather than to remove yours. + +Four sources are supported, and they differ in how well they satisfy that requirement. + +| Source | Option | m/z quality | Notes | +|---|---|---|---| +| Skyline PRISM report | `--prism-csv` | Theoretical | Recommended. Carries per-replicate RT windows. | +| DIA-NN library | `--library report-lib.parquet` | Theoretical | Needs `report.parquet` for RT windows. | +| BiblioSpec `.blib` | `--library lib.blib` | Recomputed from sequence | Requires peak annotations. | +| PRISM CSV as `--library` | `--library report.csv` | Theoretical | Same reader as `--prism-csv`. | + +## Skyline PRISM report (recommended) + +A Skyline transition report exported with the +[PRISM report template](../Skyline-PRISM-Report/Skyline-PRISM.skyr). `Product Mz` is +Skyline's computed theoretical value, which is exactly what MARS wants, and `Start Time` / +`End Time` give a real per-peptide elution window rather than a guess. + +Required columns: + +``` +Peptide Modified Sequence Unimod Ids, Precursor Charge, Precursor Mz, +Fragment Ion, Product Charge, Product Mz, Start Time, End Time +``` + +Optional: `Area` (carried as library intensity), `Retention Time`, `Protein Accession`, +`File Name`, `Replicate Name`. + +Rows whose `Fragment Ion` is `precursor` are skipped - MARS corrects fragments. + +### Replicate filtering and de-duplication + +A Skyline report lists every transition once per replicate. When several runs are processed +together, that means the same theoretical m/z appears many times over. + +MARS matches the report's `File Name` (or `Replicate Name`) against the mzML files being +processed, trying an exact base-name match first and falling back to a substring test, and +then **collapses transitions that repeat across replicates**. The copies are exact +duplicates - identical theoretical m/z, identical ion annotation - so keeping them would +multiply matching work and training rows without adding information. + +On the reference Astral plate this collapsed 1,462,106 duplicate transitions out of +2.2M rows, cutting matching work roughly threefold. Pass `--no-dedupe-library` to keep them. + +> The Python implementation does not de-duplicate, which is why its Astral run reports +> 9.1M matches where the C# reports 4.2M. Duplicating every row uniformly does not change +> what the model learns. + +Files this large stream rather than load: the reference plate report is 16.1 GB and +67,119,180 rows, read in 97 seconds. + +## DIA-NN + +Two files are needed, and they are not interchangeable: + +- **`report-lib.parquet`** - the spectral library, carrying `Product.Mz`, `Fragment.Type` + and the rest of the fragment information. +- **`report.parquet`** - the per-run identifications, carrying `RT.Start` and `RT.Stop`. + +```bash +mars calibrate --mzml-dir runs/ \ + --library out/report-lib.parquet \ + --diann-report out/report.parquet +``` + +`report.parquet` is found automatically if it sits beside the library. Handing MARS a +`report.parquet` where the library belongs is the usual mistake, and the error message says +so by name rather than listing missing columns. + +RT windows are widened across the runs being processed, so a spectrum from any of them falls +inside the window. + +> This reader has been exercised against parquet files written by the test suite, not against +> real DIA-NN output. Treat the first run against a real DIA-NN result as a check of the +> reader as much as of the data. + +### Compression codecs on Arm Windows and Intel macOS + +`Parquet.Net` delegates some codecs to a native library that is only published for +`win-x64`, `linux-x64`, `linux-arm64` and `osx-arm64`. On the other two platforms MARS +ships for - **Windows on Arm and Intel macOS** - that library is absent, and parquet falls +back to managed implementations for most codecs: + +| Codec | Without the native library | +|---|---| +| uncompressed, Snappy, Gzip, Brotli, Zstd | works | +| LZ4, LZO | fails: "No compression codec for LZ4 is available on this platform" | + +Snappy is parquet's usual default and what DIA-NN writes, so this is unlikely to bite. If +it does, the message names the codec, and the fix is to read the library on another +platform or re-export it with Snappy. Nothing else in MARS is affected: PRISM CSV, `.blib` +and every part of mzML processing are pure managed. + +## BiblioSpec (.blib) + +A `.blib` stores the **observed** m/z of each reference peak. Used directly, matching against +it measures the difference between two runs' calibration errors rather than an absolute mass +error - which is not what MARS is for. + +So for annotated peaks MARS recomputes b and y fragment m/z from the peptide sequence, using +the per-position mass deltas in the library's own `Modifications` table. That yields a real +theoretical value, modifications included. + +> The Python implementation recomputes from the *stripped* sequence, discarding +> modifications, so every fragment of a modified peptide that spans the modified residue gets +> a theoretical m/z wrong by the modification mass. + +With no `Modifications` table to read, MARS falls back to the mass deltas written into the +modified sequence itself - `M[+15.9949]` gives one, `M[Carbamidomethyl]` and `M(unimod:35)` +name a modification without saying what it weighs. Where an entry carries a modification MARS +cannot weigh, its recorded m/z is used as-is rather than recomputed, and the count of such +entries is reported. Recomputing from a sequence with a modification silently dropped does not +produce a missing answer, it produces a confident wrong one: the residue keeps its unmodified +mass and every fragment past that position is off by the delta. + +**Peaks the library does not annotate are skipped by default.** There is no way to know which +fragment ion an unannotated peak is, so its only available m/z is the observed one. A library +with hundreds of unannotated peaks per spectrum would swamp the real fragments with rows +whose label is meaningless. + +If a library has no annotations at all, MARS refuses it and names the alternatives rather +than producing a model from it. `example-data/Stellar-HeLa-GPF.blib` is such a library: its +`RefSpectraPeakAnnotations` table is empty, and running the Python implementation against it +yields 7.9M pseudo-matches from a single file and a model that reduces the spread by 2.2%. + +`--rt-window` sets the half-width of the RT window placed around each entry's library RT +(default 0.083 min, five seconds). A PRISM report's real elution windows are better when +available. + +### No native SQLite + +`.blib` is a SQLite database, but MARS reads it with a small managed reader written for this +purpose - B-tree walking, overflow page chains, record decoding - rather than +`Microsoft.Data.Sqlite`. That keeps this path free of per-platform native binaries, which is +a stated goal of the port. + +The goal is not fully met today: `Parquet.Net`, used for DIA-NN libraries, brings a native +compression library with it. It is confined to `MARS.IO`, so `MARS.Core` remains pure +managed, but a build that reads DIA-NN parquet is not native-free. Splitting the DIA-NN +reader into its own assembly would restore the property for consumers that do not need it. + +The reader is read-only and supports table B-trees, overflow chains, the record format and +UTF-8/UTF-16 text. It does not support indices, WAL or writing, none of which a library scan +needs. + +## RF temperature logs (optional) + +Two extra features become available when RF generator temperature traces are supplied: + +```bash +mars calibrate ... --temperature-dir temperature_csvs/ +``` + +Files are matched by name: `RFA2-{mzml base name}.csv` and `RFC2-{mzml base name}.csv`, +exported from Xcalibur as chromatogram CSVs. Lookups are nearest-neighbor in retention time. + +They contributed little on the reference cohort (importance below 0.01 each), so this is +worth having when the logs exist and not worth chasing when they do not. + +## Choosing a tolerance + +Usually you do not have to. MARS reads the mass analyzer out of the mzML and picks: + +| Detected | Tolerance | QC report drawn in | +|---|---|---| +| Ion trap, quadrupole | 0.3 Th | Th | +| Orbitrap, FT-ICR, TOF, Astral | 10 ppm | ppm | + +It says which in the log, on the line above the first match: + +``` +INFO high-resolution data; fragment tolerance 10 ppm (--tolerance or --tolerance-ppm to override) +``` + +Override it with `--tolerance`, `--tolerance-ppm`, or `--resolution unit|hram` - see +[the CLI reference](cli-reference.md#resolution-and-tolerance). + +**Why it matters more than it looks.** An absolute tolerance on high-resolution data is far +too wide: 0.3 Th is about 430 ppm at m/z 700, so the window is two orders of magnitude wider +than the error and the most-intense-peak rule routinely selects a different ion. A ppm +tolerance on ion-trap data is far too narrow to catch the error MARS exists to measure. +Neither mistake stops the run. Matching the Astral file below at 0.3 Th returns 3,414,802 +fragments rather than 1,408,902, and reports a standard deviation of 162 ppm against the +4.1 ppm that is really there - a complete report, all of it meaningless. That failure being +silent is why the analyzer is detected rather than documented. + +`mars qc` reports the error in both Th and ppm whichever scale it draws in, which is the +quickest way to check the tolerance is sane before training anything. diff --git a/dotnet/.gitignore b/dotnet/.gitignore new file mode 100644 index 0000000..ac8d21b --- /dev/null +++ b/dotnet/.gitignore @@ -0,0 +1,3 @@ +bin/ +obj/ +*.user diff --git a/dotnet/Directory.Build.props b/dotnet/Directory.Build.props new file mode 100644 index 0000000..edd7688 --- /dev/null +++ b/dotnet/Directory.Build.props @@ -0,0 +1,41 @@ + + + + + + net8.0;net10.0 + net8.0 + $(MarsTargetFrameworks) + + 12 + enable + disable + true + true + false + University of Washington + MARS + Copyright (c) University of Washington 2026 + 26.1.0 + $(NoWarn);CS1591 + + + diff --git a/dotnet/Directory.Build.rsp b/dotnet/Directory.Build.rsp new file mode 100644 index 0000000..ad53fe6 --- /dev/null +++ b/dotnet/Directory.Build.rsp @@ -0,0 +1,23 @@ +# Response file: MSBuild reads this for every command-line build, publish and test run from +# this directory, so nobody has to remember the flag below. +# +# Publishing MARS as a single file turns on the single-file analyzer, and that setting flows +# down into pwiz-sharp's projects when MARS is built with pwiz. There, pwiz's own +# TreatWarningsAsErrors makes IL3000 fatal. It is raised against pwiz's Waters reader, which +# MARS pulls in only transitively - Thermo references Analysis, which references Waters - and +# never calls. It is a false positive even there: WatersRawFile.AssemblyDirectory tests +# Assembly.Location for empty and falls back to AppContext.BaseDirectory, which is exactly the +# documented remedy for single-file. +# +# It has to be a GLOBAL property. The error is raised three project references down, and +# neither UndefineProperties nor AdditionalProperties on MARS.Pwiz's reference reaches that +# far - both were tried, and a publish that appeared to work turned out to be reusing a cached +# Waters build. +# +# CS1591 and CA1859 are pwiz-sharp's own suppressions, repeated because a global property +# REPLACES the value a project sets rather than adding to it. Passing IL3000 alone drops those +# two and turns one suppressed false positive into a hundred real errors. Keep this list in +# step with pwiz-sharp/Directory.Build.props. +# +# Semicolons are escaped: MSBuild would otherwise read them as a property separator. +-p:WarningsNotAsErrors=CS1591%3BCA1859%3BIL3000 diff --git a/dotnet/MARS.Core/CrossValidation.cs b/dotnet/MARS.Core/CrossValidation.cs new file mode 100644 index 0000000..11fbc97 --- /dev/null +++ b/dotnet/MARS.Core/CrossValidation.cs @@ -0,0 +1,327 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Peptide-grouped k-fold cross-validation. + +using System; +using System.Collections.Generic; +using pwiz.Osprey.ML; + +namespace MARS.Core; + +/// Accuracy of a set of predictions against their labels, all in Th. +public readonly struct FoldMetrics +{ + public required int Rows { get; init; } + + /// Median absolute residual. + public required double Mad { get; init; } + + /// Root mean squared residual. + public required double Rms { get; init; } + + /// Standard deviation of the residual. + public required double StdDev { get; init; } + + /// Median residual: what is left of any constant bias. + public required double Median { get; init; } + + /// Pearson correlation between predicted and observed error. + public required double PearsonR { get; init; } + + /// Median absolute error before correction, for the same rows. + public required double MadBefore { get; init; } + + /// Percent reduction in median absolute error. + public double MadReduction => MadBefore > 0 ? 100.0 * (1.0 - (Mad / MadBefore)) : 0.0; +} + +/// Cross-validation outcome: per-fold accuracy, and the spread across folds. +public sealed class CrossValidationReport +{ + public required int Folds { get; init; } + + public required int Groups { get; init; } + + public required FoldMetrics[] PerFold { get; init; } + + /// + /// Metrics over every row's out-of-fold prediction pooled together. This is the honest + /// headline number: every row was scored by a model that never saw its peptide. + /// + public required FoldMetrics OutOfFold { get; init; } + + /// + /// The same three measurements in ppm, or null when fragment m/z was not collected. + /// + /// + /// A mass error of 0.0013 Th means something quite different at m/z 300 and at m/z 1200, + /// and on a high-resolution instrument ppm is the scale the error is actually specified + /// in. Converted per row from each fragment's own m/z rather than by dividing an + /// aggregate by a nominal mass, which would be wrong by however wide the m/z range is. + /// + public FoldMetrics[]? PerFoldPpm { get; init; } + + public FoldMetrics? OutOfFoldPpm { get; init; } + + public FoldMetrics? InSamplePpm { get; init; } + + /// + /// The applied model scored on the rows it was fitted to. This is what the corrected + /// files will look like when re-matched. + /// + public required FoldMetrics InSample { get; init; } + + /// Standard deviation across folds of the per-fold MAD. + public double MadSpread => Spread(static m => m.Mad); + + public double RmsSpread => Spread(static m => m.Rms); + + public double PearsonRSpread => Spread(static m => m.PearsonR); + + public double MadReductionSpread => Spread(static m => m.MadReduction); + + /// + /// How much better the correction is on the data it was fitted to than on data it was + /// not. + /// + /// + /// Not a measure of cheating: calibrating a run from its own identified species is + /// exactly how mass calibration works, and the correction moves a peak onto a fitted + /// surface rather than onto its theoretical m/z, so there is little scope to memorize + /// individual peaks. What a large gap does mean is that the surface is being driven by + /// the particular peptides in this run rather than by the instrument, which says the fit + /// is thin and that reusing this model elsewhere would disappoint. + /// + public double OptimismMad => OutOfFold.Mad - InSample.Mad; + + private double Spread(Func select) => Spread(PerFold, select); + + /// + /// Standard deviation across folds of one measurement. + /// + /// + /// Public because a report drawn in ppm has to take the spread from the per-fold ppm + /// figures rather than converting the Th one: each fold converts at its own distribution + /// of fragment m/z, so a ppm spread is not a rescaling of a Th spread. + /// + public static double Spread(FoldMetrics[] folds, Func select) + { + if (folds.Length < 2) return double.NaN; + + double mean = 0; + foreach (FoldMetrics fold in folds) mean += select(fold); + mean /= folds.Length; + + double sumSquares = 0; + foreach (FoldMetrics fold in folds) + { + double d = select(fold) - mean; + sumSquares += d * d; + } + + // Sample standard deviation: the folds are a sample of the splits that could have + // been drawn, not the population of them. + return Math.Sqrt(sumSquares / (folds.Length - 1)); + } +} + +public static class PeptideFolds +{ + /// + /// Assigns each row to a fold, keeping every row of a peptide in the same fold. + /// + /// + /// Grouping is the whole point. Fragments of one peptide recur across hundreds of + /// spectra with the same theoretical m/z, and fragment_mz is a model feature, so + /// splitting a peptide across the train/test boundary lets the model memorize that + /// peptide's error rather than learn the instrument's. A row-random split reports a + /// validation error that is far better than the model will achieve on a peptide it has + /// never seen. + /// + /// Groups are sorted and dealt round-robin rather than shuffled with a PRNG. That is + /// what Osprey's Percolator implementation does + /// (PercolatorSampling.CreateStratifiedFoldsByPeptide), it is deterministic + /// without needing a seed, and it balances the number of groups per fold exactly. + /// + /// + /// Peptide group id per row. + /// Number of folds. Must be at least 2. + /// Fold index per row, and the number of distinct groups. + public static (int[] FoldOfRow, int GroupCount) AssignFolds(ReadOnlySpan groupOfRow, int folds) + { + if (folds < 2) throw new ArgumentOutOfRangeException(nameof(folds), folds, "At least 2 folds are required."); + + var rowsByGroup = new Dictionary>(); + for (int i = 0; i < groupOfRow.Length; i++) + { + if (!rowsByGroup.TryGetValue(groupOfRow[i], out List? rows)) + { + rows = new List(); + rowsByGroup[groupOfRow[i]] = rows; + } + + rows.Add(i); + } + + var groups = new int[rowsByGroup.Count]; + rowsByGroup.Keys.CopyTo(groups, 0); + Array.Sort(groups); + + var foldOfRow = new int[groupOfRow.Length]; + for (int g = 0; g < groups.Length; g++) + { + int fold = g % folds; + foreach (int row in rowsByGroup[groups[g]]) foldOfRow[row] = fold; + } + + return (foldOfRow, groups.Length); + } + + /// + /// Splits rows into train and held-out sets by peptide, for the single-fit path. + /// Returns the held-out set as close to of rows as whole + /// groups allow. + /// + public static (int[] Train, int[] Validation) SplitByGroup( + ReadOnlySpan groupOfRow, double fraction, int seed) + { + if (fraction <= 0) + { + var all = new int[groupOfRow.Length]; + for (int i = 0; i < all.Length; i++) all[i] = i; + return (all, Array.Empty()); + } + + var rowsByGroup = new Dictionary>(); + for (int i = 0; i < groupOfRow.Length; i++) + { + if (!rowsByGroup.TryGetValue(groupOfRow[i], out List? rows)) + { + rows = new List(); + rowsByGroup[groupOfRow[i]] = rows; + } + + rows.Add(i); + } + + var groups = new int[rowsByGroup.Count]; + rowsByGroup.Keys.CopyTo(groups, 0); + Array.Sort(groups); + + // Shuffled rather than dealt, because unlike k-fold this takes a prefix, and a + // sorted prefix would systematically select whichever peptides happen to sort first. + var rng = new XorShift64((ulong)seed); + for (int i = groups.Length - 1; i > 0; i--) + { + int j = (int)(rng.Next() % (ulong)(i + 1)); + (groups[i], groups[j]) = (groups[j], groups[i]); + } + + var validation = new List(); + int target = (int)Math.Round(groupOfRow.Length * fraction); + int taken = 0; + int consumed = 0; + while (consumed < groups.Length - 1 && taken < target) + { + List rows = rowsByGroup[groups[consumed]]; + validation.AddRange(rows); + taken += rows.Count; + consumed++; + } + + var train = new List(groupOfRow.Length - taken); + for (int g = consumed; g < groups.Length; g++) train.AddRange(rowsByGroup[groups[g]]); + + // Ascending row order on both sides, so downstream float accumulation depends on the + // data rather than on the shuffle. + int[] trainArray = train.ToArray(); + int[] validationArray = validation.ToArray(); + Array.Sort(trainArray); + Array.Sort(validationArray); + return (trainArray, validationArray); + } + + /// + /// Rescales a Th observation and prediction into ppm, per row, then measures. + /// + /// 1e6 divided by each row's fragment m/z. + public static FoldMetrics MeasurePpm( + ReadOnlySpan observed, ReadOnlySpan predicted, ReadOnlySpan scale) + { + var observedPpm = new double[observed.Length]; + var predictedPpm = new double[observed.Length]; + for (int i = 0; i < observed.Length; i++) + { + observedPpm[i] = observed[i] * scale[i]; + predictedPpm[i] = predicted[i] * scale[i]; + } + + return Measure(observedPpm, predictedPpm); + } + + /// Accuracy of against . + public static FoldMetrics Measure(ReadOnlySpan observed, ReadOnlySpan predicted) + { + int n = observed.Length; + if (n == 0) + { + return new FoldMetrics + { + Rows = 0, Mad = double.NaN, Rms = double.NaN, StdDev = double.NaN, + Median = double.NaN, PearsonR = double.NaN, MadBefore = double.NaN, + }; + } + + var residual = new double[n]; + double sumSquares = 0; + for (int i = 0; i < n; i++) + { + residual[i] = observed[i] - predicted[i]; + sumSquares += residual[i] * residual[i]; + } + + ErrorSummary after = MarsStatistics.Summarize(residual); + ErrorSummary before = MarsStatistics.Summarize(observed); + + return new FoldMetrics + { + Rows = n, + Mad = after.Mad, + Rms = Math.Sqrt(sumSquares / n), + StdDev = after.StdDev, + Median = after.Median, + PearsonR = Pearson(observed, predicted), + MadBefore = before.Mad, + }; + } + + private static double Pearson(ReadOnlySpan a, ReadOnlySpan b) + { + int n = a.Length; + if (n < 2) return double.NaN; + + double meanA = 0, meanB = 0; + for (int i = 0; i < n; i++) + { + meanA += a[i]; + meanB += b[i]; + } + + meanA /= n; + meanB /= n; + + double covariance = 0, varianceA = 0, varianceB = 0; + for (int i = 0; i < n; i++) + { + double da = a[i] - meanA; + double db = b[i] - meanB; + covariance += da * db; + varianceA += da * da; + varianceB += db * db; + } + + double denominator = Math.Sqrt(varianceA * varianceB); + // A constant prediction has no variance to correlate with. That is a real outcome - + // a model that learned nothing - so report it as undefined rather than as zero. + return denominator > 0 ? covariance / denominator : double.NaN; + } +} diff --git a/dotnet/MARS.Core/FragmentMatcher.cs b/dotnet/MARS.Core/FragmentMatcher.cs new file mode 100644 index 0000000..4bff8a8 --- /dev/null +++ b/dotnet/MARS.Core/FragmentMatcher.cs @@ -0,0 +1,251 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from mars/matching.py (match_library_to_spectra). + +using System; +using System.Collections.Generic; + +namespace MARS.Core; + +public sealed class MatchOptions +{ + /// Absolute matching tolerance in Th. Ignored when is positive. + public double MzToleranceTh { get; set; } = 0.3; + + /// Relative matching tolerance in ppm. Overrides when positive. + public double TolerancePpm { get; set; } + + /// Minimum observed peak intensity for a peak to be usable as a training row. + public double MinIntensity { get; set; } = 500.0; + + /// Skip spectra whose isolation window is wider than this, in Th. Null disables the filter. + public double? MaxIsolationWindowWidth { get; set; } + + /// Minimum retention time to process, in minutes. + public double? MinRetentionTime { get; set; } + + /// Maximum retention time to process, in minutes. + public double? MaxRetentionTime { get; set; } +} + +public sealed class MatchStatistics +{ + public long SpectraSeen; + + public long SpectraMatched; + + public long FragmentsMatched; + + public long CandidateFragmentsConsidered; + + public readonly SortedSet<(int Low, int High)> IsolationWindows = new(); + + public int UniqueEntriesMatched; +} + +/// +/// Matches library fragments against the peaks of one DIA MS2 spectrum and appends one +/// training row per match. +/// +public sealed class FragmentMatcher +{ + private readonly SpectralLibrary _library; + private readonly MatchOptions _options; + private readonly int[] _order; + private readonly double[] _sortedPrecursorMz; + private readonly bool[] _entryMatched; + + public FragmentMatcher(SpectralLibrary library, MatchOptions options) + { + _library = library; + _options = options; + _order = library.OrderByPrecursorMz(); + _sortedPrecursorMz = new double[_order.Length]; + for (int i = 0; i < _order.Length; i++) _sortedPrecursorMz[i] = library.PrecursorMz[_order[i]]; + _entryMatched = new bool[library.EntryCount]; + } + + public MatchStatistics Statistics { get; } = new(); + + /// + /// The features a match table must collect for this matcher's output. Temperature + /// features are only included when a temperature trace is available. + /// + public static MarsFeature[] CollectedFeatures(InjectionTimeUse injectionTime, bool rfa2, bool rfc2) + { + var features = new List(MarsFeatures.Count) + { + MarsFeature.PrecursorMz, + MarsFeature.FragmentMz, + MarsFeature.LogTic, + MarsFeature.LogIntensity, + MarsFeature.AbsoluteTime, + }; + + // Everything below needs the run to record an injection time, and nothing below needs + // it to vary. Whether it varies is decided later, from the whole matched column rather + // than from a sample of the head - see MzCalibrator.SelectFeatures. Collecting a + // column costs one array; deciding too early costs the feature. + if (injectionTime != InjectionTimeUse.Absent) + { + features.Add(MarsFeature.InjectionTime); + features.Add(MarsFeature.TicInjectionTime); + features.Add(MarsFeature.FragmentIons); + features.AddRange(MarsFeatures.NeighborFeatures); + features.AddRange(MarsFeatures.RatioFeatures); + } + + if (rfa2) features.Add(MarsFeature.Rfa2Temp); + if (rfc2) features.Add(MarsFeature.Rfc2Temp); + return features.ToArray(); + } + + /// + /// Matches one spectrum. Returns the number of rows appended to . + /// + public int MatchSpectrum(SpectrumRecord spectrum, TemperatureSet? temperatures, MatchTable table) + { + Statistics.SpectraSeen++; + + if (_options.MaxIsolationWindowWidth is double maxWidth && + spectrum.IsolationWindowWidth > maxWidth) + { + return 0; + } + + if (_options.MinRetentionTime is double minRt && spectrum.RetentionTime < minRt) return 0; + if (_options.MaxRetentionTime is double maxRt && spectrum.RetentionTime > maxRt) return 0; + + Statistics.IsolationWindows.Add(((int)spectrum.PrecursorMzLow, (int)spectrum.PrecursorMzHigh)); + + int first = PeakSearch.LowerBound(_sortedPrecursorMz, spectrum.PrecursorMzLow); + int last = PeakSearch.UpperBound(_sortedPrecursorMz, spectrum.PrecursorMzHigh); + if (first >= last) return 0; + + ReadOnlySpan mz = spectrum.Mz; + ReadOnlySpan intensity = spectrum.Intensity; + if (mz.Length == 0) return 0; + + double injectionTime = spectrum.InjectionTime ?? double.NaN; + bool hasInjectionTime = spectrum.InjectionTime.HasValue; + + double logTic = Math.Log10(Math.Max(spectrum.SummedIntensity, 1.0)); + double ticInjectionTime = hasInjectionTime ? spectrum.SummedIntensity * injectionTime : double.NaN; + + double rfa2 = temperatures?.Rfa2 is { } a ? a.TemperatureAt(spectrum.RetentionTime) : double.NaN; + double rfc2 = temperatures?.Rfc2 is { } c ? c.TemperatureAt(spectrum.RetentionTime) : double.NaN; + + double lowestUsableMz = mz[0] - MaxToleranceAt(mz[0]); + double highestUsableMz = mz[mz.Length - 1] + MaxToleranceAt(mz[mz.Length - 1]); + + int rowsAdded = 0; + Span neighbors = stackalloc double[MarsFeatures.NeighborWindows.Length]; + + for (int k = first; k < last; k++) + { + int entry = _order[k]; + + double rtStart = _library.RtStart[entry]; + double rtEnd = _library.RtEnd[entry]; + if (!double.IsNaN(rtStart) && !double.IsNaN(rtEnd)) + { + if (spectrum.RetentionTime < rtStart || spectrum.RetentionTime > rtEnd) continue; + } + + int fragmentStart = _library.FragmentStart[entry]; + int fragmentEnd = _library.FragmentStart[entry + 1]; + + for (int f = fragmentStart; f < fragmentEnd; f++) + { + double expectedMz = _library.FragmentMz[f]; + if (expectedMz <= 0) continue; + if (expectedMz < lowestUsableMz || expectedMz > highestUsableMz) continue; + + Statistics.CandidateFragmentsConsidered++; + + if (!PeakSearch.TryFindMostIntensePeak( + expectedMz, mz, intensity, + _options.MzToleranceTh, _options.MinIntensity, _options.TolerancePpm, + out double observedMz, out double observedIntensity)) + { + continue; + } + + table.Set(MarsFeature.PrecursorMz, spectrum.PrecursorMzCenter); + table.Set(MarsFeature.FragmentMz, expectedMz); + table.Set(MarsFeature.LogTic, logTic); + table.Set(MarsFeature.LogIntensity, Math.Log10(Math.Max(observedIntensity, 1.0))); + table.Set(MarsFeature.AbsoluteTime, spectrum.AbsoluteTime); + + if (table.Has(MarsFeature.InjectionTime)) + { + table.Set(MarsFeature.InjectionTime, injectionTime); + table.Set(MarsFeature.TicInjectionTime, ticInjectionTime); + } + + if (table.Has(MarsFeature.FragmentIons)) + { + double fragmentIons = hasInjectionTime ? observedIntensity * injectionTime : double.NaN; + table.Set(MarsFeature.FragmentIons, fragmentIons); + + for (int w = 0; w < neighbors.Length; w++) + { + if (!hasInjectionTime) + { + neighbors[w] = double.NaN; + continue; + } + + (double low, double high) = MarsFeatures.NeighborWindows[w]; + neighbors[w] = PeakSearch.SumIntensityInRange( + mz, intensity, expectedMz + low, expectedMz + high) * injectionTime; + } + + for (int w = 0; w < neighbors.Length; w++) + table.Set(MarsFeatures.NeighborFeatures[w], neighbors[w]); + + // The Python implementation leaves the ratios undefined (and therefore + // drops the row) when the fragment ion count is not strictly positive. + bool ratiosDefined = fragmentIons > 0; + for (int w = 0; w < neighbors.Length; w++) + { + table.Set( + MarsFeatures.RatioFeatures[w], + ratiosDefined ? neighbors[w] / fragmentIons : double.NaN); + } + } + + if (table.Has(MarsFeature.Rfa2Temp)) table.Set(MarsFeature.Rfa2Temp, rfa2); + if (table.Has(MarsFeature.Rfc2Temp)) table.Set(MarsFeature.Rfc2Temp, rfc2); + + table.DeltaMz.Add(observedMz - expectedMz); + table.ObservedIntensity.Add(observedIntensity); + table.PeptideGroup.Add(_library.PeptideGroup[entry]); + + if (table.KeepDetail) + { + table.ScanNumber!.Add(spectrum.ScanNumber); + table.LibraryEntryIndex!.Add(entry); + table.FragmentIndex!.Add(f); + table.ObservedMz!.Add(observedMz); + table.RetentionTime!.Add(spectrum.RetentionTime); + } + + table.CommitRow(); + rowsAdded++; + + Statistics.FragmentsMatched++; + if (!_entryMatched[entry]) + { + _entryMatched[entry] = true; + Statistics.UniqueEntriesMatched++; + } + } + } + + if (rowsAdded > 0) Statistics.SpectraMatched++; + return rowsAdded; + } + + private double MaxToleranceAt(double mz) => + _options.TolerancePpm > 0 ? mz * _options.TolerancePpm / 1e6 : _options.MzToleranceTh; +} diff --git a/dotnet/MARS.Core/ISpectrumSource.cs b/dotnet/MARS.Core/ISpectrumSource.cs new file mode 100644 index 0000000..c91a3ff --- /dev/null +++ b/dotnet/MARS.Core/ISpectrumSource.cs @@ -0,0 +1,70 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; + +namespace MARS.Core; + +/// +/// A run MARS can read spectra from, whatever file format it arrives in. +/// +/// +/// +/// MARS was built around mzML, and everything above this interface still is: the matcher, the +/// feature extraction and the model all consume and neither know +/// nor care where it came from. This exists so a Thermo .raw can be read directly, +/// without a conversion step whose only purpose was to produce something MARS could open. +/// +/// +/// Implementations are in the assemblies that own the format - mzML in MARS.IO, vendor +/// formats in MARS.Pwiz - so that MARS.Core depends on neither. +/// +/// +public interface ISpectrumSource : IDisposable +{ + /// Path of the file or directory backing this source. + string Path { get; } + + /// Size in bytes, for reporting. Zero when it cannot be determined cheaply. + long Length { get; } + + /// + /// Run start as a Unix timestamp in seconds, or null when the file does not record one. + /// The absolute_time feature is undefined without it. + /// + double? AcquisitionStartTime { get; } + + /// + /// The analyzer that recorded this run's MS2 spectra, which decides the default fragment + /// tolerance and the units the QC report is drawn in. + /// + /// + /// MS2 specifically. On a hybrid instrument that is not the analyzer the run names as its + /// default: an Orbitrap Astral file declares the orbitrap, which takes the MS1 survey, + /// and points only its MS2 spectra at the Astral analyzer. + /// + MassAnalyzerClass Analyzer { get; } + + /// + /// Streams spectra at the given MS level, or every spectrum when null. + /// + /// + /// The arrays on the yielded record may be reused between iterations, so a consumer that + /// keeps them must copy. This is what lets MARS hold a 4.9 GB run in a bounded working + /// set. + /// + IEnumerable ReadSpectra(int? msLevel = 2); +} + +/// What a run's ion injection times look like, and so whether they can be a feature. +public enum InjectionTimeUse +{ + /// The run does not record one. + Absent, + + /// Recorded, but the same on every spectrum, so it carries no information. + Constant, + + /// Recorded and varying, as a trap's gain control makes it. + Varying, +} diff --git a/dotnet/MARS.Core/MARS.Core.csproj b/dotnet/MARS.Core/MARS.Core.csproj new file mode 100644 index 0000000..b08505e --- /dev/null +++ b/dotnet/MARS.Core/MARS.Core.csproj @@ -0,0 +1,17 @@ + + + + MARS.Core + MARS.Core + MARS calibration model, feature extraction and fragment matching. + + + + + + + + + + + diff --git a/dotnet/MARS.Core/MarsFeature.cs b/dotnet/MARS.Core/MarsFeature.cs new file mode 100644 index 0000000..8d51455 --- /dev/null +++ b/dotnet/MARS.Core/MarsFeature.cs @@ -0,0 +1,243 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Feature definitions transcribed from mars/matching.py and +// MzCalibrator._prepare_features in mars/calibration.py. + +using System; +using System.Collections.Generic; + +namespace MARS.Core; + +/// +/// The MARS feature vocabulary, in the canonical order produced by the Python +/// MzCalibrator._prepare_features. The enum values ARE the model's feature order; +/// a model file records the active subset by name, and loading a model whose name list +/// does not match the extractor is a hard error. Do not reorder without a format bump. +/// +public enum MarsFeature +{ + /// Isolation window target m/z of the DIA window, Th. + PrecursorMz = 0, + + /// Theoretical library m/z when training; the observed peak m/z when correcting. + FragmentMz = 1, + + /// log10(max(summed spectrum intensity, 1)). + LogTic = 2, + + /// log10(max(peak intensity, 1)). + LogIntensity = 3, + + /// Seconds since the earliest acquisition start across the processed files. + AbsoluteTime = 4, + + /// Ion injection time, seconds. + InjectionTime = 5, + + /// Summed spectrum intensity multiplied by injection time. + TicInjectionTime = 6, + + /// Peak intensity multiplied by injection time; an ion count rather than a rate. + FragmentIons = 7, + + /// Injection-time-scaled intensity in (x + 0.5, x + 1.5] Th. + IonsAbove01 = 8, + + /// Injection-time-scaled intensity in (x + 1.5, x + 2.5] Th. + IonsAbove12 = 9, + + /// Injection-time-scaled intensity in (x + 2.5, x + 3.5] Th. + IonsAbove23 = 10, + + /// Injection-time-scaled intensity in (x - 1.5, x - 0.5] Th. + IonsBelow01 = 11, + + /// Injection-time-scaled intensity in (x - 2.5, x - 1.5] Th. + IonsBelow12 = 12, + + /// Injection-time-scaled intensity in (x - 3.5, x - 2.5] Th. + IonsBelow23 = 13, + + /// IonsAbove01 divided by FragmentIons, or 0 when FragmentIons is not positive. + AdjacentRatio01 = 14, + + AdjacentRatio12 = 15, + + AdjacentRatio23 = 16, + + AdjacentRatioBelow01 = 17, + + AdjacentRatioBelow12 = 18, + + AdjacentRatioBelow23 = 19, + + /// RFA2 RF-generator temperature at this retention time, degrees C. + Rfa2Temp = 20, + + /// RFC2 RF-generator temperature at this retention time, degrees C. + Rfc2Temp = 21, +} + +public static class MarsFeatures +{ + /// Total size of the feature vocabulary. + public const int Count = 22; + + /// + /// Feature names, index-aligned with . These strings are the + /// on-disk contract with the model file and match the Python column names exactly. + /// + public static readonly string[] Names = + { + "precursor_mz", + "fragment_mz", + "log_tic", + "log_intensity", + "absolute_time", + "injection_time", + "tic_injection_time", + "fragment_ions", + "ions_above_0_1", + "ions_above_1_2", + "ions_above_2_3", + "ions_below_0_1", + "ions_below_1_2", + "ions_below_2_3", + "adjacent_ratio_0_1", + "adjacent_ratio_1_2", + "adjacent_ratio_2_3", + "adjacent_ratio_below_0_1", + "adjacent_ratio_below_1_2", + "adjacent_ratio_below_2_3", + "rfa2_temp", + "rfc2_temp", + }; + + /// + /// Neighbor-density window bounds in Th relative to the reference m/z x, as + /// (lowExclusive, highInclusive]. Index-aligned with IonsAbove01..IonsBelow23. + /// + public static readonly (double Low, double High)[] NeighborWindows = + { + (0.5, 1.5), + (1.5, 2.5), + (2.5, 3.5), + (-1.5, -0.5), + (-2.5, -1.5), + (-3.5, -2.5), + }; + + /// The six neighbor-density features, in canonical order. + public static readonly MarsFeature[] NeighborFeatures = + { + MarsFeature.IonsAbove01, + MarsFeature.IonsAbove12, + MarsFeature.IonsAbove23, + MarsFeature.IonsBelow01, + MarsFeature.IonsBelow12, + MarsFeature.IonsBelow23, + }; + + /// The six ratio features, index-aligned with . + public static readonly MarsFeature[] RatioFeatures = + { + MarsFeature.AdjacentRatio01, + MarsFeature.AdjacentRatio12, + MarsFeature.AdjacentRatio23, + MarsFeature.AdjacentRatioBelow01, + MarsFeature.AdjacentRatioBelow12, + MarsFeature.AdjacentRatioBelow23, + }; + + public static string NameOf(MarsFeature feature) => Names[(int)feature]; + + public static bool TryParse(string name, out MarsFeature feature) + { + for (int i = 0; i < Names.Length; i++) + { + if (string.Equals(Names[i], name, StringComparison.Ordinal)) + { + feature = (MarsFeature)i; + return true; + } + } + + feature = default; + return false; + } + + /// + /// True when the feature is only defined once the ion injection time is known. + /// The Python implementation drops this whole group when no spectrum reports one. + /// + public static bool RequiresInjectionTime(MarsFeature feature) => + feature >= MarsFeature.InjectionTime && feature <= MarsFeature.AdjacentRatioBelow23; +} + +/// +/// The ordered subset of the vocabulary a particular model was trained on. The Python +/// implementation selects this subset at fit time from which columns carry data. +/// +public sealed class FeatureSet +{ + private readonly int[] _slotOf; // vocabulary index -> column index, or -1 + + public FeatureSet(IReadOnlyList features) + { + Features = new MarsFeature[features.Count]; + for (int i = 0; i < features.Count; i++) Features[i] = features[i]; + + _slotOf = new int[MarsFeatures.Count]; + for (int i = 0; i < _slotOf.Length; i++) _slotOf[i] = -1; + for (int i = 0; i < Features.Length; i++) + { + int v = (int)Features[i]; + if (_slotOf[v] >= 0) + throw new ArgumentException("Duplicate feature: " + MarsFeatures.Names[v], nameof(features)); + _slotOf[v] = i; + } + } + + public MarsFeature[] Features { get; } + + public int Count => Features.Length; + + /// Column index of a feature in this set, or -1 when it is not present. + public int SlotOf(MarsFeature feature) => _slotOf[(int)feature]; + + public bool Contains(MarsFeature feature) => _slotOf[(int)feature] >= 0; + + /// True when any neighbor-density or ratio feature is active. + public bool NeedsNeighborDensity + { + get + { + for (int i = 0; i < MarsFeatures.NeighborFeatures.Length; i++) + { + if (Contains(MarsFeatures.NeighborFeatures[i])) return true; + if (Contains(MarsFeatures.RatioFeatures[i])) return true; + } + + return false; + } + } + + public string[] Names() + { + var names = new string[Features.Length]; + for (int i = 0; i < Features.Length; i++) names[i] = MarsFeatures.NameOf(Features[i]); + return names; + } + + public static FeatureSet FromNames(IReadOnlyList names) + { + var features = new List(names.Count); + foreach (string name in names) + { + if (!MarsFeatures.TryParse(name, out MarsFeature f)) + throw new ArgumentException("Unknown MARS feature name: " + name, nameof(names)); + features.Add(f); + } + + return new FeatureSet(features); + } +} diff --git a/dotnet/MARS.Core/MarsModelFile.cs b/dotnet/MARS.Core/MarsModelFile.cs new file mode 100644 index 0000000..14677a2 --- /dev/null +++ b/dotnet/MARS.Core/MarsModelFile.cs @@ -0,0 +1,335 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Versioned, self-describing model file. Replaces the Python pickle, which could not be +// read outside a matching Python + xgboost install. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; +using pwiz.Osprey.ML; + +namespace MARS.Core; + +/// On-disk representation of a trained MARS model. +public sealed class MarsModelFile +{ + /// + /// Bumped whenever the meaning of a field changes. A reader that does not recognize + /// the version refuses the file rather than guessing. + /// + /// + /// Version 2 added the cross-validation summary. The model itself is still one object: + /// a cross-validated fit merges its fold models into one rather than storing them + /// separately, so version 1 files load unchanged. + /// + public const int CurrentFormatVersion = 2; + + public int FormatVersion { get; set; } = CurrentFormatVersion; + + public string MarsVersion { get; set; } = MarsInfo.Version; + + /// Feature names in model order. Loading fails if any name is unknown. + public List FeatureNames { get; set; } = new(); + + /// Seconds subtracted from raw acquisition timestamps before training. + public double AbsoluteTimeOffset { get; set; } + + public CalibrationOptionsDto Options { get; set; } = new(); + + /// + /// The model. After cross-validation this is the fold models merged into one, which + /// predicts identically to averaging them. + /// + public GbtModelDto Model { get; set; } = new(); + + public TrainingSummaryDto? Training { get; set; } + + /// Cross-validation summary, or null when a single model was fitted. + public CrossValidationDto? CrossValidation { get; set; } + + /// + /// What the fold models scored on the peptides they did not train on. Recorded so a + /// model file states its own honest accuracy rather than only the accuracy it achieved + /// on the rows it was built from. + /// + public sealed class CrossValidationDto + { + public int Folds { get; set; } + + /// Distinct peptides the folds were dealt over. + public int Groups { get; set; } + + public double OutOfFoldMad { get; set; } + + public double OutOfFoldRms { get; set; } + + public double OutOfFoldPearsonR { get; set; } + + public double InSampleMad { get; set; } + + /// Per-fold median absolute residual, one entry per fold. + public double[] FoldMad { get; set; } = Array.Empty(); + + /// Standard deviation across folds of the per-fold MAD. + public double MadSpread { get; set; } + } + + public sealed class CalibrationOptionsDto + { + public int NEstimators { get; set; } + + public int MaxDepth { get; set; } + + public double LearningRate { get; set; } + + public double MinChildWeight { get; set; } + + public double Subsample { get; set; } + + public double ColSampleByTree { get; set; } + + public double Gamma { get; set; } + + public double RegLambda { get; set; } + + public double RegAlpha { get; set; } + + public int MaxBins { get; set; } + + public int Seed { get; set; } + + public double ValidationSplit { get; set; } + + public bool WeightByIntensity { get; set; } + } + + public sealed class GbtModelDto + { + public double BaseScore { get; set; } + + /// + /// Feature-vector width the ensemble was trained on. Always equal to the length of + /// , and written anyway so the tree arrays + /// are self-describing to a reader that only looks at this object. + /// + public int FeatureCount { get; set; } + + /// Loss the ensemble was fitted under. MARS always uses squared error. + public GbtObjective Objective { get; set; } = GbtObjective.SquaredError; + + public int[] Feature { get; set; } = Array.Empty(); + + public double[] Threshold { get; set; } = Array.Empty(); + + public int[] Left { get; set; } = Array.Empty(); + + public int[] Right { get; set; } = Array.Empty(); + + public double[] Leaf { get; set; } = Array.Empty(); + + public int[] TreeRoot { get; set; } = Array.Empty(); + } + + public sealed class TrainingSummaryDto + { + public int RowsMatched { get; set; } + + public int RowsUsed { get; set; } + + public int RowsTrain { get; set; } + + public int RowsValidation { get; set; } + + public double TrainMae { get; set; } + + public double TrainRmse { get; set; } + + public double ValidationMae { get; set; } + + public double ValidationRmse { get; set; } + + public double BeforeStdDev { get; set; } + + public double AfterStdDev { get; set; } + + public double BeforeMad { get; set; } + + public double AfterMad { get; set; } + + public double[] PermutationImportance { get; set; } = Array.Empty(); + + public int[] SplitCount { get; set; } = Array.Empty(); + } +} + +public static class MarsInfo +{ + /// + /// MARS version, read from the assembly rather than declared here, so + /// dotnet/Directory.Build.props is the single place a release bumps. A duplicated + /// literal is the kind of thing that stays correct until exactly the release where it + /// does not, and it is stamped into every model file MARS writes. + /// + public static string Version { get; } = ReadAssemblyVersion(); + + private static string ReadAssemblyVersion() + { + Assembly assembly = typeof(MarsInfo).Assembly; + string? informational = assembly + .GetCustomAttribute()?.InformationalVersion; + + if (string.IsNullOrEmpty(informational)) + return assembly.GetName().Version?.ToString(3) ?? "0.0.0"; + + // The SDK appends "+" when source link is on; the build metadata is + // not part of the version a user should see. + int plus = informational.IndexOf('+'); + return plus < 0 ? informational : informational[..plus]; + } +} + +public static class MarsModelIo +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals, + }; + + private static MarsModelFile.GbtModelDto ToDto(GradientBoostedTrees model) + { + GbtModelData data = model.ToModelData(); + return new MarsModelFile.GbtModelDto + { + BaseScore = data.BaseScore, + FeatureCount = data.FeatureCount, + Objective = data.Objective, + Feature = data.Feature, + Threshold = data.Threshold, + Left = data.Left, + Right = data.Right, + Leaf = data.Leaf, + TreeRoot = data.TreeRoot, + }; + } + + public static void Save(MzCalibrator calibrator, string path) + { + TrainingStatistics? stats = calibrator.Statistics; + + var file = new MarsModelFile + { + FeatureNames = new List(calibrator.Features.Names()), + AbsoluteTimeOffset = calibrator.AbsoluteTimeOffset, + Options = new MarsModelFile.CalibrationOptionsDto + { + NEstimators = calibrator.Options.NEstimators, + MaxDepth = calibrator.Options.MaxDepth, + LearningRate = calibrator.Options.LearningRate, + MinChildWeight = calibrator.Options.MinChildWeight, + Subsample = calibrator.Options.Subsample, + ColSampleByTree = calibrator.Options.ColSampleByTree, + Gamma = calibrator.Options.Gamma, + RegLambda = calibrator.Options.RegLambda, + RegAlpha = calibrator.Options.RegAlpha, + MaxBins = calibrator.Options.MaxBins, + Seed = calibrator.Options.Seed, + ValidationSplit = calibrator.Options.ValidationSplit, + WeightByIntensity = calibrator.Options.WeightByIntensity, + }, + Model = ToDto(calibrator.Model), + CrossValidation = calibrator.CrossValidation is not CrossValidationReport cv + ? null + : new MarsModelFile.CrossValidationDto + { + Folds = cv.Folds, + Groups = cv.Groups, + OutOfFoldMad = cv.OutOfFold.Mad, + OutOfFoldRms = cv.OutOfFold.Rms, + OutOfFoldPearsonR = cv.OutOfFold.PearsonR, + InSampleMad = cv.InSample.Mad, + FoldMad = Array.ConvertAll(cv.PerFold, static f => f.Mad), + MadSpread = cv.MadSpread, + }, + Training = stats is null ? null : new MarsModelFile.TrainingSummaryDto + { + RowsMatched = stats.RowsMatched, + RowsUsed = stats.RowsUsed, + RowsTrain = stats.RowsTrain, + RowsValidation = stats.RowsValidation, + TrainMae = stats.TrainMae, + TrainRmse = stats.TrainRmse, + ValidationMae = stats.ValidationMae, + ValidationRmse = stats.ValidationRmse, + BeforeStdDev = stats.Before.StdDev, + AfterStdDev = stats.After.StdDev, + BeforeMad = stats.Before.Mad, + AfterMad = stats.After.Mad, + PermutationImportance = stats.PermutationImportance, + SplitCount = stats.SplitCount, + }, + }; + + string? directory = Path.GetDirectoryName(Path.GetFullPath(path)); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + using FileStream stream = File.Create(path); + JsonSerializer.Serialize(stream, file, SerializerOptions); + } + + public static MzCalibrator Load(string path) + { + using FileStream stream = File.OpenRead(path); + MarsModelFile? file = JsonSerializer.Deserialize(stream, SerializerOptions); + if (file is null) throw new InvalidDataException($"Model file is empty or malformed: {path}"); + + if (file.FormatVersion != MarsModelFile.CurrentFormatVersion) + { + throw new InvalidDataException( + $"Model file '{path}' is format version {file.FormatVersion}; this build reads version {MarsModelFile.CurrentFormatVersion}."); + } + + // A model whose feature list does not match the extractor's vocabulary is a hard + // error, not a warning: silently scoring a differently shaped row would corrupt + // every m/z it touched. + FeatureSet features = FeatureSet.FromNames(file.FeatureNames); + + GradientBoostedTrees model = GradientBoostedTrees.FromModelData(new GbtModelData + { + BaseScore = file.Model.BaseScore, + + // A file written before these two fields existed carries the same information in + // its feature name list, and MARS only ever fits squared error, so derive rather + // than reject. + FeatureCount = file.Model.FeatureCount > 0 ? file.Model.FeatureCount : features.Count, + Objective = file.Model.Objective, + Feature = file.Model.Feature, + Threshold = file.Model.Threshold, + Left = file.Model.Left, + Right = file.Model.Right, + Leaf = file.Model.Leaf, + TreeRoot = file.Model.TreeRoot, + }); + + var options = new CalibrationOptions + { + NEstimators = file.Options.NEstimators, + MaxDepth = file.Options.MaxDepth, + LearningRate = file.Options.LearningRate, + MinChildWeight = file.Options.MinChildWeight, + Subsample = file.Options.Subsample, + ColSampleByTree = file.Options.ColSampleByTree, + Gamma = file.Options.Gamma, + RegLambda = file.Options.RegLambda, + RegAlpha = file.Options.RegAlpha, + MaxBins = file.Options.MaxBins, + Seed = file.Options.Seed, + ValidationSplit = file.Options.ValidationSplit, + WeightByIntensity = file.Options.WeightByIntensity, + }; + + return new MzCalibrator(features, model, file.AbsoluteTimeOffset, options, null, null); + } +} diff --git a/dotnet/MARS.Core/MassAnalyzer.cs b/dotnet/MARS.Core/MassAnalyzer.cs new file mode 100644 index 0000000..07b7b44 --- /dev/null +++ b/dotnet/MARS.Core/MassAnalyzer.cs @@ -0,0 +1,126 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; + +namespace MARS.Core; + +/// How precisely the analyzer that recorded the MS2 spectra measures m/z. +public enum MassAnalyzerClass +{ + /// Nothing in the file said, so MARS should not assume. + Unknown, + + /// An ion trap or quadrupole: error is roughly constant in Th. + UnitResolution, + + /// Orbitrap, FT-ICR, TOF, Astral: error is roughly constant in ppm. + HighResolution, +} + +/// +/// Classifies the mass analyzer that produced a run's MS2 spectra, from the CV accessions +/// mzML records for it. +/// +/// +/// +/// This decides two things a user otherwise has to know and pass by hand: whether the +/// fragment tolerance should default to Th or ppm, and which of those the QC report should +/// be drawn in. Getting it wrong is not a cosmetic problem. A 0.3 Th window is about 430 ppm +/// at m/z 700, so running the trap default against Astral data widens the window by two +/// orders of magnitude, and the extra width fills with wrong matches. The run still +/// completes and still reports numbers, which is what makes it worth detecting rather than +/// documenting. +/// +/// +/// By accession, not by name: names are display strings that differ between writers, and new +/// analyzers arrive faster than the writers agree on what to call them. +/// +/// +public static class MassAnalyzers +{ + public const string Quadrupole = "MS:1000081"; + public const string IonTrap = "MS:1000264"; + public const string QuadrupoleIonTrap = "MS:1000082"; + public const string RadialEjectionLinearIonTrap = "MS:1000083"; + public const string AxialEjectionLinearIonTrap = "MS:1000078"; + public const string LinearIonTrap = "MS:1000291"; + + public const string Orbitrap = "MS:1000484"; + public const string FourierTransformIonCyclotronResonance = "MS:1000079"; + public const string TimeOfFlight = "MS:1000084"; + + /// The Astral analyzer, added to the CV in 2023. + public const string AsymmetricTrackLosslessTimeOfFlight = "MS:1003379"; + + private static readonly HashSet Unit = new(StringComparer.Ordinal) + { + Quadrupole, IonTrap, QuadrupoleIonTrap, RadialEjectionLinearIonTrap, + AxialEjectionLinearIonTrap, LinearIonTrap, + }; + + private static readonly HashSet HighResolution = new(StringComparer.Ordinal) + { + Orbitrap, FourierTransformIonCyclotronResonance, TimeOfFlight, + AsymmetricTrackLosslessTimeOfFlight, + }; + + public static MassAnalyzerClass Classify(string? accession) + { + if (accession is null) return MassAnalyzerClass.Unknown; + if (HighResolution.Contains(accession)) return MassAnalyzerClass.HighResolution; + if (Unit.Contains(accession)) return MassAnalyzerClass.UnitResolution; + return MassAnalyzerClass.Unknown; + } + + /// + /// Classifies from a Thermo filter string, for files whose instrument configuration is + /// missing or unrecognized. ITMS, FTMS and ASTMS are the analyzer + /// tokens Thermo writes at the front of every filter. + /// + public static MassAnalyzerClass ClassifyFilterString(string? filter) + { + if (string.IsNullOrEmpty(filter)) return MassAnalyzerClass.Unknown; + + if (filter.StartsWith("ITMS", StringComparison.Ordinal)) return MassAnalyzerClass.UnitResolution; + if (filter.StartsWith("FTMS", StringComparison.Ordinal) || + filter.StartsWith("ASTMS", StringComparison.Ordinal) || + filter.StartsWith("TOFMS", StringComparison.Ordinal)) + { + return MassAnalyzerClass.HighResolution; + } + + return MassAnalyzerClass.Unknown; + } + + /// + /// The analyzer a configuration measures with: the highest-order component. A + /// configuration lists its analyzers in beam order, so an Astral configuration is + /// quadrupole at order 2 and the Astral analyzer at order 3, and it is the last one that + /// determines the mass accuracy. + /// + public static string? MeasuringAnalyzer(IReadOnlyList<(int Order, string Accession)> analyzers) + { + // Highest order wins, but a quadrupole only wins if there is nothing else: in a + // configuration like the Astral's it is the isolating element rather than the + // measuring one, and classifying on it would call the run unit-resolution. + string? best = null; + int bestOrder = int.MinValue; + foreach ((int order, string accession) in analyzers) + { + bool preferable = best is null || + (best == Quadrupole && accession != Quadrupole) || + (order > bestOrder && !(accession == Quadrupole && best != Quadrupole)); + if (preferable) (best, bestOrder) = (accession, order); + } + + return best; + } + + public static string Describe(MassAnalyzerClass analyzer) => analyzer switch + { + MassAnalyzerClass.HighResolution => "high-resolution", + MassAnalyzerClass.UnitResolution => "unit-resolution", + _ => "unknown", + }; +} diff --git a/dotnet/MARS.Core/MatchTable.cs b/dotnet/MARS.Core/MatchTable.cs new file mode 100644 index 0000000..d700544 --- /dev/null +++ b/dotnet/MARS.Core/MatchTable.cs @@ -0,0 +1,222 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Column store for the fragment matches that become training rows. + +using System; +using System.Collections.Generic; + +namespace MARS.Core; + +/// Growable array that hands back its backing store without a final copy. +public sealed class GrowableArray +{ + private T[] _items; + + public GrowableArray(int capacity = 1024) => _items = new T[Math.Max(4, capacity)]; + + public int Count { get; private set; } + + public T[] Items => _items; + + public T this[int i] + { + get => _items[i]; + set => _items[i] = value; + } + + public void Add(T value) + { + if (Count == _items.Length) Array.Resize(ref _items, _items.Length * 2); + _items[Count++] = value; + } + + /// Discards the most recently added values. + public void Truncate(int n) => Count -= n; + + public T[] ToArray() + { + var copy = new T[Count]; + Array.Copy(_items, copy, Count); + return copy; + } +} + +/// +/// The matched-fragment table, stored column-major so that a 9-million-row Astral plate +/// fits without a managed object per row. One row is one library fragment matched to one +/// observed peak in one spectrum. +/// +public sealed class MatchTable +{ + private readonly GrowableArray?[] _columns = new GrowableArray?[MarsFeatures.Count]; + + public MatchTable(IReadOnlyList collect, bool keepDetail = false) + { + Collected = new MarsFeature[collect.Count]; + for (int i = 0; i < collect.Count; i++) + { + Collected[i] = collect[i]; + _columns[(int)collect[i]] = new GrowableArray(); + } + + KeepDetail = keepDetail; + if (keepDetail) + { + ScanNumber = new GrowableArray(); + LibraryEntryIndex = new GrowableArray(); + FragmentIndex = new GrowableArray(); + ObservedMz = new GrowableArray(); + RetentionTime = new GrowableArray(); + } + } + + public MarsFeature[] Collected { get; } + + public bool KeepDetail { get; } + + public int Count { get; private set; } + + /// Label: observed m/z minus theoretical library m/z, in Th. + public GrowableArray DeltaMz { get; } = new(); + + /// Sample weight source: intensity of the matched peak. + public GrowableArray ObservedIntensity { get; } = new(); + + /// + /// Peptide identity of the library entry this row came from. Cross-validation folds + /// are assigned over this so a peptide never straddles a train/test boundary; it is + /// always collected, since the split depends on it whether or not anything else does. + /// + public GrowableArray PeptideGroup { get; } = new(); + + public GrowableArray? ScanNumber { get; } + + public GrowableArray? LibraryEntryIndex { get; } + + /// Index into the library's flat fragment arrays, identifying which fragment + /// of the entry matched. With the scan number this is a unique key for a row, which is + /// what lets a dump be joined against another implementation's output. + public GrowableArray? FragmentIndex { get; } + + public GrowableArray? ObservedMz { get; } + + public GrowableArray? RetentionTime { get; } + + public bool Has(MarsFeature feature) => _columns[(int)feature] is not null; + + /// Backing column for a feature. Only valid for collected features. + public GrowableArray Column(MarsFeature feature) => + _columns[(int)feature] ?? throw new InvalidOperationException( + "Feature not collected: " + MarsFeatures.NameOf(feature)); + + public void Set(MarsFeature feature, double value) => _columns[(int)feature]?.Add(value); + + /// Commits the values staged by Set / DeltaMz.Add for the current row. + public void CommitRow() => Count++; + + /// + /// Adds a constant offset to a column across every row. Used to re-base absolute_time + /// once the earliest acquisition across all input files is known. + /// + public void OffsetColumn(MarsFeature feature, double offset) + { + GrowableArray? column = _columns[(int)feature]; + if (column is null) return; + double[] values = column.Items; + for (int i = 0; i < column.Count; i++) values[i] += offset; + } + + public double MinOf(MarsFeature feature) + { + GrowableArray? column = _columns[(int)feature]; + if (column is null || column.Count == 0) return double.NaN; + double[] values = column.Items; + double min = double.PositiveInfinity; + for (int i = 0; i < column.Count; i++) + { + if (values[i] < min) min = values[i]; + } + + return min; + } + + public double MaxOf(MarsFeature feature) + { + GrowableArray? column = _columns[(int)feature]; + if (column is null || column.Count == 0) return double.NaN; + double[] values = column.Items; + double max = double.NegativeInfinity; + for (int i = 0; i < column.Count; i++) + { + if (values[i] > max) max = values[i]; + } + + return max; + } + + /// True when at least one row has a finite value in this column. + public bool AnyFinite(MarsFeature feature) + { + GrowableArray? column = _columns[(int)feature]; + if (column is null) return false; + double[] values = column.Items; + for (int i = 0; i < column.Count; i++) + { + if (double.IsFinite(values[i])) return true; + } + + return false; + } + + /// + /// True when this column's finite values are not all the same. + /// + /// + /// Asked of the whole matched column rather than a sample of it. An ion trap holds its + /// injection time at the method's ceiling for as long as the trap does not fill, which on + /// a gradient means the entire void volume - tens of thousands of spectra before anything + /// elutes. A run sampled only at the head therefore looks constant no matter how much it + /// varies later, and on a Stellar it varies a great deal: a standard 4 m/z DIA run of HeLa + /// carries 65,059 distinct injection times over 97,500 MS2, two thirds of them off the + /// ceiling, with the first at spectrum 9,253. + /// + public bool Varies(MarsFeature feature) + { + GrowableArray? column = _columns[(int)feature]; + if (column is null) return false; + + double[] values = column.Items; + double first = double.NaN; + double low = double.MaxValue; + double high = double.MinValue; + + for (int i = 0; i < column.Count; i++) + { + double v = values[i]; + if (!double.IsFinite(v)) continue; + if (double.IsNaN(first)) first = v; + if (v < low) low = v; + if (v > high) high = v; + } + + if (double.IsNaN(first)) return false; + + // Relative, so it is not a statement about the units. Orders of magnitude tighter + // than any real gain control - a trap's injection times differ by whole milliseconds. + double scale = Math.Abs(high) > 0 ? Math.Abs(high) : 1.0; + return (high - low) / scale > 1e-6; + } + + /// True when every row has a finite value in this column. + public bool AllFinite(MarsFeature feature) + { + GrowableArray? column = _columns[(int)feature]; + if (column is null) return false; + double[] values = column.Items; + for (int i = 0; i < column.Count; i++) + { + if (double.IsNaN(values[i])) return false; + } + + return true; + } +} diff --git a/dotnet/MARS.Core/MzCalibrator.cs b/dotnet/MARS.Core/MzCalibrator.cs new file mode 100644 index 0000000..c60465b --- /dev/null +++ b/dotnet/MARS.Core/MzCalibrator.cs @@ -0,0 +1,938 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from mars/calibration.py (MzCalibrator). + +using System; +using System.Collections.Generic; +using pwiz.Osprey.ML; + +namespace MARS.Core; + +/// How the second pass handles rows the first pass could not explain. +public enum RobustFit +{ + /// Fit once. Every row counts the same, however implausible its label. + None, + + /// Drop rows beyond the threshold and fit again. + Trim, + + /// + /// Down-weight rows in proportion to how far past the threshold they sit, and fit again. + /// Gentler than , and on the reference data slightly worse: a row a + /// little past the threshold keeps nearly all of its weight. + /// + Huber, +} + +/// +/// Hyperparameters, transcribed from the Python MzCalibrator, which constructs +/// xgboost.XGBRegressor(n_estimators=100, max_depth=6, learning_rate=0.1, +/// random_state=42, objective="reg:squarederror") and leaves the rest at the XGBoost +/// library default. +/// +public sealed class CalibrationOptions +{ + public int NEstimators { get; set; } = 100; + + public int MaxDepth { get; set; } = 6; + + public double LearningRate { get; set; } = 0.1; + + /// + /// Under squared error the hessian is the sample weight, so this is a sample count. + /// XGBoost's default of 1 carries over directly because reg:squarederror gives it the + /// same meaning there. + /// + public double MinChildWeight { get; set; } = 1.0; + + public double Subsample { get; set; } = 1.0; + + public double ColSampleByTree { get; set; } = 1.0; + + public double Gamma { get; set; } + + public double RegLambda { get; set; } = 1.0; + + public double RegAlpha { get; set; } + + public int MaxBins { get; set; } = 256; + + public int Seed { get; set; } = 42; + + /// Fraction of rows held out to report validation error. 0 disables the split. + public double ValidationSplit { get; set; } = 0.2; + + /// + /// Cross-validation folds. 2 or more trains one model per fold and makes the calibrator + /// their ensemble; 0 or 1 falls back to a single fit with a held-out split. + /// + /// + /// Folds are assigned by peptide, never by row, so every reported number comes from a + /// model that never saw the peptide it is scoring. Cross-validation costs one training + /// round per fold, which is a minority of a run's wall clock because matching dominates. + /// + public int CvFolds { get; set; } = 5; + + + /// + /// Weight training rows by observed peak intensity, normalized to mean 1. More intense + /// fragments give a better-determined centroid, so they should count for more. + /// + public bool WeightByIntensity { get; set; } = true; + + /// Histogram threads. Determinism holds at any value; see Osprey.ML. + public int MaxDegreeOfParallelism { get; set; } = -1; + + /// + /// Cap on training rows, applied by deterministic stride subsampling. 0 means no cap, + /// which is what the Python implementation does. + /// + public int MaxTrainingRows { get; set; } + + /// Rows sampled when estimating permutation importance. 0 disables it. + public int ImportanceSampleRows { get; set; } = 50000; + + /// How the second pass treats rows the first pass could not explain. + /// + /// Trim rather than Huber, on measurement. Huber is the more principled choice when + /// outliers are extreme measurements of the right quantity; here they are measurements + /// of the wrong one - the most intense peak in the window was a different ion - so the + /// label carries no information at all and softening its influence is not enough. At + /// three robust sigma, Huber still leaves such a row 79% of its weight on average. + /// + public RobustFit Robust { get; set; } = RobustFit.Trim; + + /// + /// Residual threshold for the second pass, in robust standard deviations. 0 disables it. + /// + /// + /// + /// Matching takes the most intense peak within the tolerance window, and sometimes that + /// peak is not the fragment. Those rows carry a delta that is not a mass error at all - + /// on the reference Stellar run they are three times weaker than the rest, sit in + /// spectra with a quarter as many fragment ions, and cluster against the edge of the + /// window - and squared error is exactly the loss that lets them pull the fit. + /// + /// + /// The threshold is in units of a MAD-derived sigma, so it adapts to the instrument + /// rather than assuming a Th value. Trimming applies to TRAINING rows only. Held-out + /// rows are always scored in full, or the reported accuracy would improve simply by + /// discarding the hard cases from the measurement. + /// + /// + public double RobustSigma { get; set; } = 3.0; +} + +public sealed class TrainingStatistics +{ + public int RowsMatched { get; init; } + + public int RowsUsed { get; init; } + + public int RowsTrain { get; init; } + + public int RowsValidation { get; init; } + + public double TrainMae { get; init; } + + public double TrainRmse { get; init; } + + public double ValidationMae { get; init; } + + public double ValidationRmse { get; init; } + + public ErrorSummary Before { get; init; } + + public ErrorSummary After { get; init; } + + /// Before and after in ppm, or null when fragment m/z was not collected. + public ErrorSummary? BeforePpm { get; init; } + + public ErrorSummary? AfterPpm { get; init; } + + /// Permutation importance per active feature, normalized to sum to 1. + public double[] PermutationImportance { get; init; } = Array.Empty(); + + /// Number of splits made on each active feature. + public int[] SplitCount { get; init; } = Array.Empty(); +} + +/// +/// The m/z calibration model: predicts the mass error of a peak from its spectral context, +/// so the corrected value is observed - PredictDelta(features). +/// +public sealed class MzCalibrator +{ + internal MzCalibrator( + FeatureSet features, + GradientBoostedTrees model, + double absoluteTimeOffset, + CalibrationOptions options, + TrainingStatistics? statistics, + CrossValidationReport? crossValidation) + { + Features = features; + Model = model; + AbsoluteTimeOffset = absoluteTimeOffset; + Options = options; + Statistics = statistics; + CrossValidation = crossValidation; + } + + public FeatureSet Features { get; } + + /// + /// The model that corrects the data, fitted to every usable row. + /// + /// + /// Calibration is in-sample by nature, so nothing is withheld from the surface being + /// fitted. holds the separate question of whether that + /// surface is real structure and what it would achieve on a run it was not fitted to. + /// + public GradientBoostedTrees Model { get; } + + /// Cross-validation results, or null when a single model was fitted. + public CrossValidationReport? CrossValidation { get; } + + /// + /// Seconds subtracted from every raw acquisition timestamp to produce the + /// the model was trained on: the earliest + /// acquisition start across the training files. + /// + /// This has to travel with the model. The Python implementation re-bases absolute_time + /// to the earliest run before fitting, but feeds the raw Unix timestamp back in when it + /// writes the corrected file, so every inference row lands far above the largest + /// training value and the feature degenerates to a constant branch. Carrying the offset + /// keeps training and inference on the same scale. + /// + /// + public double AbsoluteTimeOffset { get; } + + public CalibrationOptions Options { get; } + + public TrainingStatistics? Statistics { get; } + + /// Predicted mass error in Th. Subtract from the observed m/z to correct it. + public double PredictDelta(double[] featureRow) => Model.ScoreSingle(featureRow); + + /// + /// Predicted mass error for every row of a match table, parallel to the table's rows. + /// A row with any undefined feature scores NaN rather than being silently dropped, so + /// the result lines up with the table and with a dump of it. + /// + /// + /// This exists so the learned function can be compared against another implementation + /// on identical rows. Comparing two boosting implementations tree by tree is not + /// meaningful; comparing what they predict for the same input is. + /// + public double[] PredictAll(MatchTable table) + { + int featureCount = Features.Count; + var columns = new double[featureCount][]; + for (int j = 0; j < featureCount; j++) + columns[j] = table.Column(Features.Features[j]).Items; + + var predictions = new double[table.Count]; + var row = new double[featureCount]; + + for (int i = 0; i < table.Count; i++) + { + bool usable = true; + for (int j = 0; j < featureCount; j++) + { + double value = columns[j][i]; + if (double.IsNaN(value)) + { + usable = false; + break; + } + + row[j] = value; + } + + predictions[i] = usable ? PredictDelta(row) : double.NaN; + } + + return predictions; + } + + /// + /// Fits a calibrator on matched fragments. + /// + /// Matched fragments, one row per library fragment matched to a peak. + /// Hyperparameters. + /// + /// Earliest acquisition start across the input files, already subtracted from the + /// table's column. + /// + /// Optional progress sink. + public static MzCalibrator Fit( + MatchTable table, + CalibrationOptions options, + double absoluteTimeOffset, + Action? log = null) + { + if (table.Count == 0) + throw new InvalidOperationException("No fragment matches: nothing to train on."); + + FeatureSet features = SelectFeatures(table, log); + int[] rows = SelectRows(table, features, options, log); + if (rows.Length == 0) + throw new InvalidOperationException("Every candidate training row had a missing feature value."); + + int nFeat = features.Count; + var x = new double[rows.Length][]; + var y = new double[rows.Length]; + double[]? weights = options.WeightByIntensity ? new double[rows.Length] : null; + + double[] deltaMz = table.DeltaMz.Items; + double[] intensity = table.ObservedIntensity.Items; + var columns = new double[nFeat][]; + for (int j = 0; j < nFeat; j++) columns[j] = table.Column(features.Features[j]).Items; + + double weightSum = 0; + for (int i = 0; i < rows.Length; i++) + { + int r = rows[i]; + var row = new double[nFeat]; + for (int j = 0; j < nFeat; j++) row[j] = columns[j][r]; + x[i] = row; + y[i] = deltaMz[r]; + if (weights is not null) + { + weights[i] = intensity[r]; + weightSum += intensity[r]; + } + } + + if (weights is not null) + { + // The Python implementation normalizes weights to mean 1. That matters here: + // min_child_weight thresholds the summed hessian, which under squared error is + // the summed weight, so raw detector counts would make the threshold meaningless. + double meanWeight = weightSum / rows.Length; + if (meanWeight > 0) + { + for (int i = 0; i < weights.Length; i++) weights[i] /= meanWeight; + } + } + + // Fold assignment needs each row's peptide, so a peptide's fragments cannot be split + // across a train/test boundary. Without this the model can memorize a peptide's + // fragment m/z values and every held-out number comes out optimistic. + if (table.PeptideGroup.Count != table.Count) + { + throw new InvalidOperationException( + $"The match table has {table.Count:N0} rows but {table.PeptideGroup.Count:N0} " + + "peptide group values. Every row needs one: folds and the held-out split are " + + "assigned over peptides, and falling back to row-random splitting would report " + + "an accuracy the model cannot reach on an unseen peptide."); + } + + var groupOfRow = new int[rows.Length]; + int[] groupColumn = table.PeptideGroup.Items; + for (int i = 0; i < rows.Length; i++) groupOfRow[i] = groupColumn[rows[i]]; + + // Per row, from that fragment's own m/z. Dividing an aggregate by a nominal mass + // would be wrong by however wide the cohort's m/z range is, which on a plasma + // digest is most of a factor of four. + double[]? ppmScale = null; + if (table.Has(MarsFeature.FragmentMz)) + { + double[] fragmentMz = table.Column(MarsFeature.FragmentMz).Items; + ppmScale = new double[rows.Length]; + for (int i = 0; i < rows.Length; i++) + { + double mz = fragmentMz[rows[i]]; + ppmScale[i] = mz > 0 ? 1e6 / mz : 0.0; + } + } + + var gbtParams = new GbtParams + { + Objective = GbtObjective.SquaredError, + NTrees = options.NEstimators, + MaxDepth = options.MaxDepth, + LearningRate = options.LearningRate, + MinChildWeight = options.MinChildWeight, + Subsample = options.Subsample, + ColSample = options.ColSampleByTree, + Gamma = options.Gamma, + RegLambda = options.RegLambda, + RegAlpha = options.RegAlpha, + MaxBins = options.MaxBins, + Seed = (ulong)options.Seed, + MaxDegreeOfParallelism = options.MaxDegreeOfParallelism <= 0 + ? Environment.ProcessorCount + : options.MaxDegreeOfParallelism, + }; + + if (options.CvFolds >= 2) + { + return FitCrossValidated( + features, x, y, weights, groupOfRow, ppmScale, gbtParams, options, + absoluteTimeOffset, table.Count, log); + } + + (int[] trainIndex, int[] validationIndex) = + PeptideFolds.SplitByGroup(groupOfRow, options.ValidationSplit, options.Seed); + + log?.Invoke( + $"Training on {trainIndex.Length:N0} rows, holding out {validationIndex.Length:N0} " + + $"by peptide, {nFeat} features"); + + (GradientBoostedTrees model, int affected) = TrainRobust( + Gather(x, trainIndex), Gather(y, trainIndex), + weights is null ? null : Gather(weights, trainIndex), gbtParams, + options.Robust, options.RobustSigma); + + if (affected > 0) + { + log?.Invoke( + $" {DescribeRobust(options.Robust)} {affected:N0} unexplainable rows " + + $"({100.0 * affected / trainIndex.Length:F1}%) and refit"); + } + + var calibrator = new MzCalibrator(features, model, absoluteTimeOffset, options, null, null); + TrainingStatistics statistics = calibrator.Evaluate( + x, y, trainIndex, validationIndex, ppmScale, table.Count, options); + return new MzCalibrator(features, model, absoluteTimeOffset, options, statistics, null); + } + + /// + /// Trains one model per fold and returns their ensemble. + /// + /// + /// The ensemble is the model, not a stepping stone to one. Osprey's Percolator does the + /// same on its tree path: the linear path can average fold weight vectors because a dot + /// product is linear in the weights, but trees cannot be averaged that way, so it + /// averages the fold SCORES instead (see PercolatorResults.FoldGbtModels and + /// PercolatorScorer.AverageGbtScore in ProteoWizard). Averaging K models trained + /// on overlapping data is also steadier than any one of them, and costs no extra + /// training round. + /// + /// Unlike Percolator, no cross-fold score calibration is needed. An SVM margin means + /// nothing across folds until it is calibrated; MARS predicts a mass error in Th, the + /// same physical quantity in every fold. + /// + /// + private static MzCalibrator FitCrossValidated( + FeatureSet features, double[][] x, double[] y, double[]? weights, int[] groupOfRow, + double[]? ppmScale, GbtParams gbtParams, CalibrationOptions options, + double absoluteTimeOffset, int matchedRows, Action? log) + { + (int[] foldOfRow, int groupCount) = PeptideFolds.AssignFolds(groupOfRow, options.CvFolds); + + if (groupCount < options.CvFolds) + { + throw new InvalidOperationException( + $"Only {groupCount} distinct peptides matched, which cannot be split into " + + $"{options.CvFolds} folds. Lower --cv-folds, or pass --cv-folds 0 to train a " + + "single model."); + } + + log?.Invoke( + $"Cross-validating: {options.CvFolds} folds over {groupCount:N0} peptides, " + + $"{x.Length:N0} rows, {features.Count} features"); + + var models = new GradientBoostedTrees[options.CvFolds]; + var perFold = new FoldMetrics[options.CvFolds]; + FoldMetrics[]? perFoldPpm = null; + var outOfFold = new double[x.Length]; + int affectedTotal = 0; + + for (int fold = 0; fold < options.CvFolds; fold++) + { + var trainRows = new List(x.Length); + var heldOutRows = new List((x.Length / options.CvFolds) + 1); + for (int i = 0; i < foldOfRow.Length; i++) + { + if (foldOfRow[i] == fold) heldOutRows.Add(i); + else trainRows.Add(i); + } + + int[] trainIndex = trainRows.ToArray(); + int[] heldOutIndex = heldOutRows.ToArray(); + + // Trim within the fold's own training rows. The held-out rows are scored in + // full: dropping the hard ones from the measurement as well would improve the + // reported number without improving anything real. + (models[fold], int foldAffected) = TrainRobust( + Gather(x, trainIndex), Gather(y, trainIndex), + weights is null ? null : Gather(weights, trainIndex), gbtParams, + options.Robust, options.RobustSigma); + affectedTotal += foldAffected; + + var heldOutObserved = new double[heldOutIndex.Length]; + var heldOutPredicted = new double[heldOutIndex.Length]; + for (int i = 0; i < heldOutIndex.Length; i++) + { + int r = heldOutIndex[i]; + heldOutObserved[i] = y[r]; + heldOutPredicted[i] = models[fold].ScoreSingle(x[r]); + outOfFold[r] = heldOutPredicted[i]; + } + + perFold[fold] = PeptideFolds.Measure(heldOutObserved, heldOutPredicted); + if (ppmScale is not null) + { + var heldOutScale = new double[heldOutIndex.Length]; + for (int i = 0; i < heldOutIndex.Length; i++) heldOutScale[i] = ppmScale[heldOutIndex[i]]; + (perFoldPpm ??= new FoldMetrics[options.CvFolds])[fold] = + PeptideFolds.MeasurePpm(heldOutObserved, heldOutPredicted, heldOutScale); + } + log?.Invoke( + $" fold {fold + 1}/{options.CvFolds}: trained on {trainIndex.Length:N0}, " + + $"scored {heldOutIndex.Length:N0}, MAD {perFold[fold].Mad:F4} Th " + + $"({perFold[fold].MadReduction:F1}% reduction), r {perFold[fold].PearsonR:F4}"); + } + + // Merge the folds into one model. This is not a refit and not an approximation: a + // boosted ensemble's score is linear in its trees, so keeping every tree and dividing + // each leaf by the fold count reproduces the average of the fold models exactly. What + // gets applied is therefore precisely what was measured, as a single object. + // The model that corrects the data is fitted to ALL of it. Calibration is in-sample + // by nature - it is how mass calibration has always worked, measuring known species + // present in the run and correcting the axis from them - so there is no reason to + // withhold data from the surface being fitted. The fold models exist to answer a + // different question, which is whether that surface is real structure or noise, and + // what it would achieve on a run it was not fitted to. + if (affectedTotal > 0) + { + log?.Invoke( + $" folds {DescribeRobust(options.Robust)} " + + $"{affectedTotal / options.CvFolds:N0} unexplainable rows each, on average"); + } + + log?.Invoke($" fitting the correction model on all {x.Length:N0} rows"); + (GradientBoostedTrees model, int affected) = TrainRobust( + x, y, weights, gbtParams, options.Robust, options.RobustSigma); + + if (affected > 0) + { + log?.Invoke( + $" {DescribeRobust(options.Robust)} {affected:N0} unexplainable rows " + + $"({100.0 * affected / x.Length:F1}%) and refit"); + } + var calibrator = new MzCalibrator(features, model, absoluteTimeOffset, options, null, null); + + var inSample = new double[x.Length]; + for (int i = 0; i < x.Length; i++) inSample[i] = calibrator.PredictDelta(x[i]); + + var report = new CrossValidationReport + { + Folds = options.CvFolds, + Groups = groupCount, + PerFold = perFold, + OutOfFold = PeptideFolds.Measure(y, outOfFold), + InSample = PeptideFolds.Measure(y, inSample), + PerFoldPpm = perFoldPpm, + OutOfFoldPpm = ppmScale is null ? null : PeptideFolds.MeasurePpm(y, outOfFold, ppmScale), + InSamplePpm = ppmScale is null ? null : PeptideFolds.MeasurePpm(y, inSample, ppmScale), + }; + + log?.Invoke( + $" on this data: MAD {report.InSample.Mad:F4} Th " + + $"({report.InSample.MadReduction:F1}% reduction)"); + log?.Invoke( + $" expected on new data: MAD {report.OutOfFold.Mad:F4} Th " + + $"({report.OutOfFold.MadReduction:F1}% reduction), r {report.OutOfFold.PearsonR:F4}, " + + $"fold spread {report.MadSpread:F4} Th"); + + TrainingStatistics statistics = + calibrator.EvaluateCrossValidated(x, y, inSample, ppmScale, matchedRows, report, options); + + return new MzCalibrator(features, model, absoluteTimeOffset, options, statistics, report); + } + + /// + /// Fits a model, then fits again with the rows the first pass could not explain either + /// removed or held down. + /// + /// + /// + /// is a Huber loss, reached by reweighting rather than by + /// changing the objective. Huber's gradient is the residual clipped to the threshold, + /// clip(r, +/-d) = r * min(1, d/|r|), and squared error on weights + /// w * min(1, d/|r|) produces exactly that gradient. So one extra pass of the + /// existing squared-error path gives the robust fit, with no new objective to add, + /// validate and keep bit-identical upstream. + /// + /// + /// Compared with it is the same idea made continuous: a row + /// two thresholds out counts half as much rather than either fully or not at all, so + /// there is no cliff for a row to sit astride, and a genuinely large but real error is + /// still heard. + /// + /// + /// Weights are renormalized to mean 1 afterwards, because min_child_weight and + /// reg_lambda are thresholds on summed weights; without it, down-weighting the + /// tail would quietly tighten both. + /// + /// + /// The model, and how many rows the second pass removed or down-weighted. + private static (GradientBoostedTrees Model, int Affected) TrainRobust( + double[][] x, double[] y, double[]? weights, GbtParams gbtParams, + RobustFit mode, double sigma) + { + GradientBoostedTrees first = GradientBoostedTrees.Train(x, y, gbtParams, weights); + if (mode == RobustFit.None || !(sigma > 0) || x.Length < 100) return (first, 0); + + var residual = new double[x.Length]; + for (int i = 0; i < x.Length; i++) residual[i] = y[i] - first.ScoreSingle(x[i]); + + // A MAD-derived scale rather than a standard deviation: the outliers being looked for + // would inflate a standard deviation enough to hide themselves. + ErrorSummary summary = MarsStatistics.Summarize(residual); + double scale = summary.Mad * 1.4826; + if (!(scale > 0)) return (first, 0); + + double limit = sigma * scale; + + if (mode == RobustFit.Trim) + { + var keep = new List(x.Length); + for (int i = 0; i < x.Length; i++) + { + if (Math.Abs(residual[i] - summary.Median) <= limit) keep.Add(i); + } + + int trimmed = x.Length - keep.Count; + + // Refitting on a much smaller set would be a different model rather than a + // cleaned one, so an unexpectedly aggressive trim is declined instead of applied. + if (trimmed == 0 || keep.Count < x.Length / 2) return (first, 0); + + int[] kept = keep.ToArray(); + return ( + GradientBoostedTrees.Train( + Gather(x, kept), Gather(y, kept), gbtParams, + weights is null ? null : Gather(weights, kept)), + trimmed); + } + + var robust = new double[x.Length]; + double sum = 0; + int held = 0; + for (int i = 0; i < x.Length; i++) + { + double excess = Math.Abs(residual[i] - summary.Median); + double factor = excess > limit ? limit / excess : 1.0; + if (factor < 1.0) held++; + robust[i] = (weights is null ? 1.0 : weights[i]) * factor; + sum += robust[i]; + } + + if (held == 0) return (first, 0); + + double mean = sum / x.Length; + if (mean > 0) + { + for (int i = 0; i < robust.Length; i++) robust[i] /= mean; + } + + return (GradientBoostedTrees.Train(x, y, gbtParams, robust), held); + } + + /// + /// Applies Python's feature-availability rules: the four always-on features, then each + /// optional feature that has at least one row carrying a value. The injection-time group + /// stands or falls together because none of it is defined without an injection time. + /// + private static FeatureSet SelectFeatures(MatchTable table, Action? log) + { + var active = new List(MarsFeatures.Count); + foreach (MarsFeature feature in new[] + { + MarsFeature.PrecursorMz, MarsFeature.FragmentMz, MarsFeature.LogTic, MarsFeature.LogIntensity, + }) + { + if (table.Has(feature)) active.Add(feature); + else log?.Invoke($"Required feature '{MarsFeatures.NameOf(feature)}' was not collected"); + } + + if (table.AnyFinite(MarsFeature.AbsoluteTime)) active.Add(MarsFeature.AbsoluteTime); + + // The injection time is a feature only where it moves. On an instrument that + // accumulates for a fixed period it never does, and then it is a constant a tree + // cannot split on, while tic_injection_time is log_tic rescaled - a duplicate that + // splits permutation importance with the feature it duplicates. + // + // Decided here, over every matched row, rather than from a sample taken before + // matching. A trap sits at its ceiling until the trap actually fills, so any run + // judged on its first few hundred spectra reads as constant whatever it does later. + if (table.AnyFinite(MarsFeature.InjectionTime)) + { + if (table.Varies(MarsFeature.InjectionTime)) + { + active.Add(MarsFeature.InjectionTime); + AddIfPresent(table, active, MarsFeature.TicInjectionTime); + } + else + { + log?.Invoke( + " ion injection time is the same on every matched spectrum; injection_time " + + "and tic_injection_time are off. The ion-population features stay: a " + + "constant scales them without flattening them."); + } + } + + // Kept whenever they were collected and hold anything, which is the matcher's decision + // rather than this one. They are scaled by the injection time but are not it: a run + // with a constant injection time still has a varying peak neighbourhood, and dropping + // these along with the injection time costs most of the correction on exactly the + // instruments that accumulate for a fixed period. + AddIfPresent(table, active, MarsFeature.FragmentIons); + foreach (MarsFeature f in MarsFeatures.NeighborFeatures) AddIfPresent(table, active, f); + foreach (MarsFeature f in MarsFeatures.RatioFeatures) AddIfPresent(table, active, f); + + AddIfPresent(table, active, MarsFeature.Rfa2Temp); + AddIfPresent(table, active, MarsFeature.Rfc2Temp); + + var set = new FeatureSet(active); + log?.Invoke($"Using {set.Count} features: {string.Join(", ", set.Names())}"); + return set; + } + + private static void AddIfPresent(MatchTable table, List active, MarsFeature feature) + { + if (table.AnyFinite(feature)) active.Add(feature); + } + + private static string DescribeRobust(RobustFit mode) => + mode == RobustFit.Trim ? "dropped" : "held down"; + + private static T[] Gather(T[] source, int[] index) + { + var result = new T[index.Length]; + for (int i = 0; i < index.Length; i++) result[i] = source[index[i]]; + return result; + } + + /// + /// Keeps rows whose selected features are all finite, then applies the optional row cap + /// by stride so the retained rows stay spread across the whole run rather than + /// clustering at the start. + /// + private static int[] SelectRows(MatchTable table, FeatureSet features, CalibrationOptions options, Action? log) + { + int n = table.Count; + var columns = new double[features.Count][]; + for (int j = 0; j < features.Count; j++) columns[j] = table.Column(features.Features[j]).Items; + double[] deltaMz = table.DeltaMz.Items; + + var kept = new List(n); + for (int i = 0; i < n; i++) + { + if (!double.IsFinite(deltaMz[i])) continue; + bool ok = true; + for (int j = 0; j < columns.Length; j++) + { + if (!double.IsFinite(columns[j][i])) + { + ok = false; + break; + } + } + + if (ok) kept.Add(i); + } + + if (kept.Count < n) + log?.Invoke($"Dropped {n - kept.Count:N0} rows with a missing feature value ({kept.Count:N0} retained)"); + + if (options.MaxTrainingRows > 0 && kept.Count > options.MaxTrainingRows) + { + var capped = new int[options.MaxTrainingRows]; + for (int i = 0; i < capped.Length; i++) + capped[i] = kept[(int)((long)i * kept.Count / capped.Length)]; + log?.Invoke($"Capped training rows at {capped.Length:N0} of {kept.Count:N0} by even stride"); + return capped; + } + + return kept.ToArray(); + } + + /// + /// Training statistics for the cross-validated path. + /// + /// + /// After is the residual of the model that will actually correct the files, on + /// the rows it was fitted to. That is what the corrected output will look like when it + /// is re-matched, and it is what a user asking "what did this do to my data" is asking + /// about. The out-of-fold figure answers the other question - what the same procedure + /// would achieve on a run it was not fitted to, which is what mars apply does - + /// and lives on . Both are reported, labelled. + /// + private TrainingStatistics EvaluateCrossValidated( + double[][] x, double[] y, double[] inSample, double[]? ppmScale, int rowsMatched, + CrossValidationReport report, CalibrationOptions options) + { + var residual = new double[y.Length]; + double absolute = 0, squares = 0; + for (int i = 0; i < y.Length; i++) + { + residual[i] = y[i] - inSample[i]; + absolute += Math.Abs(residual[i]); + squares += residual[i] * residual[i]; + } + + return new TrainingStatistics + { + RowsMatched = rowsMatched, + RowsUsed = y.Length, + RowsTrain = y.Length, + RowsValidation = report.OutOfFold.Rows, + TrainMae = y.Length > 0 ? absolute / y.Length : double.NaN, + TrainRmse = y.Length > 0 ? Math.Sqrt(squares / y.Length) : double.NaN, + + // The validation figures are the out-of-fold ones: the honest estimate for a run + // this model was not fitted to. + ValidationMae = report.OutOfFold.Rms > 0 ? OutOfFoldMae(y, report) : double.NaN, + ValidationRmse = report.OutOfFold.Rms, + Before = MarsStatistics.Summarize(y), + After = MarsStatistics.Summarize(residual), + BeforePpm = ppmScale is null ? null : SummarizePpm(y, ppmScale), + AfterPpm = ppmScale is null ? null : SummarizePpm(residual, ppmScale), + PermutationImportance = ComputePermutationImportance(x, y, options), + SplitCount = ComputeSplitCounts(), + }; + } + + private static ErrorSummary SummarizePpm(double[] values, double[] scale) + { + var ppm = new double[values.Length]; + for (int i = 0; i < values.Length; i++) ppm[i] = values[i] * scale[i]; + return MarsStatistics.Summarize(ppm); + } + + private static double OutOfFoldMae(double[] y, CrossValidationReport report) => + // RMS is carried directly; MAE is not, and recomputing it would need the predictions + // again. The median absolute residual is the figure actually reported everywhere. + report.OutOfFold.Mad; + + private TrainingStatistics Evaluate( + double[][] x, + double[] y, + int[] trainIndex, + int[] validationIndex, + double[]? ppmScale, + int rowsMatched, + CalibrationOptions options) + { + var residualsTrain = new double[trainIndex.Length]; + for (int i = 0; i < trainIndex.Length; i++) + { + int r = trainIndex[i]; + residualsTrain[i] = y[r] - PredictDelta(x[r]); + } + + var residualsValidation = new double[validationIndex.Length]; + for (int i = 0; i < validationIndex.Length; i++) + { + int r = validationIndex[i]; + residualsValidation[i] = y[r] - PredictDelta(x[r]); + } + + var after = new double[x.Length]; + for (int i = 0; i < x.Length; i++) after[i] = y[i] - PredictDelta(x[i]); + + return new TrainingStatistics + { + RowsMatched = rowsMatched, + RowsUsed = x.Length, + RowsTrain = trainIndex.Length, + RowsValidation = validationIndex.Length, + TrainMae = MarsStatistics.MeanAbsolute(residualsTrain), + TrainRmse = MarsStatistics.Rms(residualsTrain), + ValidationMae = validationIndex.Length > 0 ? MarsStatistics.MeanAbsolute(residualsValidation) : double.NaN, + ValidationRmse = validationIndex.Length > 0 ? MarsStatistics.Rms(residualsValidation) : double.NaN, + Before = MarsStatistics.Summarize(y), + After = MarsStatistics.Summarize(after), + BeforePpm = ppmScale is null ? null : SummarizePpm(y, ppmScale), + AfterPpm = ppmScale is null ? null : SummarizePpm(after, ppmScale), + PermutationImportance = ComputePermutationImportance(x, y, options), + SplitCount = ComputeSplitCounts(), + }; + } + + /// + /// Permutation importance: the increase in RMSE when one feature's values are shuffled + /// across rows, normalized to sum to 1. + /// + /// This is NOT XGBoost's gain importance, which the Python implementation reports. + /// Osprey.ML does not retain per-split gain, and permutation importance answers the + /// question people actually ask of these numbers -- how much does this feature carry -- + /// without depending on tree internals. Values are not comparable to the Python ones + /// term by term; the ranking is. + /// + /// + private double[] ComputePermutationImportance(double[][] x, double[] y, CalibrationOptions options) + { + int nFeat = Features.Count; + var importance = new double[nFeat]; + if (options.ImportanceSampleRows <= 0 || x.Length == 0) return importance; + + int sampleSize = Math.Min(options.ImportanceSampleRows, x.Length); + var sample = new double[sampleSize][]; + var target = new double[sampleSize]; + for (int i = 0; i < sampleSize; i++) + { + int r = (int)((long)i * x.Length / sampleSize); + sample[i] = (double[])x[r].Clone(); + target[i] = y[r]; + } + + double baseline = RootMeanSquareResidual(sample, target); + + var rng = new XorShift64((ulong)options.Seed + 977UL); + var scratch = new double[sampleSize]; + double total = 0; + for (int j = 0; j < nFeat; j++) + { + for (int i = 0; i < sampleSize; i++) scratch[i] = sample[i][j]; + + for (int i = sampleSize - 1; i > 0; i--) + { + int k = (int)(rng.Next() % (ulong)(i + 1)); + (sample[i][j], sample[k][j]) = (sample[k][j], sample[i][j]); + } + + double shuffled = RootMeanSquareResidual(sample, target); + importance[j] = Math.Max(0.0, shuffled - baseline); + total += importance[j]; + + for (int i = 0; i < sampleSize; i++) sample[i][j] = scratch[i]; + } + + if (total > 0) + { + for (int j = 0; j < nFeat; j++) importance[j] /= total; + } + + return importance; + } + + private double RootMeanSquareResidual(double[][] x, double[] y) + { + double sum = 0; + for (int i = 0; i < x.Length; i++) + { + double residual = y[i] - PredictDelta(x[i]); + sum += residual * residual; + } + + return Math.Sqrt(sum / x.Length); + } + + private int[] ComputeSplitCounts() + { + var counts = new int[Features.Count]; + foreach (int feature in Model.ToModelData().Feature) + { + if (feature >= 0 && feature < counts.Length) counts[feature]++; + } + + return counts; + } +} diff --git a/dotnet/MARS.Core/PeakSearch.cs b/dotnet/MARS.Core/PeakSearch.cs new file mode 100644 index 0000000..c732d40 --- /dev/null +++ b/dotnet/MARS.Core/PeakSearch.cs @@ -0,0 +1,146 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from mars/matching.py (find_most_intense_peak, sum_intensity_in_range) and +// the vectorized range-sum helper in mars/calibration.py. + +using System; + +namespace MARS.Core; + +public static class PeakSearch +{ + /// + /// numpy searchsorted(side="left"): first index whose value is greater than or equal + /// to . + /// + public static int LowerBound(ReadOnlySpan sorted, double value) + { + int lo = 0, hi = sorted.Length; + while (lo < hi) + { + int mid = (int)(((uint)lo + (uint)hi) >> 1); + if (sorted[mid] < value) lo = mid + 1; + else hi = mid; + } + + return lo; + } + + /// + /// numpy searchsorted(side="right"): first index whose value is strictly greater than + /// . + /// + public static int UpperBound(ReadOnlySpan sorted, double value) + { + int lo = 0, hi = sorted.Length; + while (lo < hi) + { + int mid = (int)(((uint)lo + (uint)hi) >> 1); + if (sorted[mid] <= value) lo = mid + 1; + else hi = mid; + } + + return lo; + } + + /// + /// Finds the most intense peak within tolerance of . + /// Ties go to the lowest m/z, matching numpy argmax. + /// + /// Absolute tolerance in Th; ignored when tolerancePpm is set. + /// Relative tolerance in ppm; overrides toleranceTh when positive. + /// True when a peak was found, with its m/z and intensity. + public static bool TryFindMostIntensePeak( + double targetMz, + ReadOnlySpan mz, + ReadOnlySpan intensity, + double toleranceTh, + double minIntensity, + double tolerancePpm, + out double observedMz, + out double observedIntensity) + { + observedMz = 0; + observedIntensity = 0; + if (mz.Length == 0) return false; + + double tolerance = tolerancePpm > 0 ? targetMz * tolerancePpm / 1e6 : toleranceTh; + int low = LowerBound(mz, targetMz - tolerance); + int high = UpperBound(mz, targetMz + tolerance); + if (low >= high) return false; + + int best = -1; + double bestIntensity = double.NegativeInfinity; + for (int i = low; i < high; i++) + { + double value = intensity[i]; + if (minIntensity > 0 && value < minIntensity) continue; + if (value > bestIntensity) + { + bestIntensity = value; + best = i; + } + } + + if (best < 0) return false; + + observedMz = mz[best]; + observedIntensity = intensity[best]; + return true; + } + + /// + /// Sums intensities of peaks in the half-open interval (lowMz, highMz]: peaks exactly at + /// lowMz are excluded, peaks exactly at highMz are included. Matches the Python + /// searchsorted(side="right") on both bounds. + /// + public static double SumIntensityInRange( + ReadOnlySpan mz, + ReadOnlySpan intensity, + double lowMz, + double highMz) + { + if (mz.Length == 0) return 0.0; + + int low = UpperBound(mz, lowMz); + int high = UpperBound(mz, highMz); + if (low >= high) return 0.0; + + double sum = 0.0; + for (int i = low; i < high; i++) sum += intensity[i]; + return sum; + } + + /// + /// Computes, for every peak i, the summed intensity in (mz[i] + low, mz[i] + high]. + /// + /// Both interval ends are monotone in i because mz is ascending, so the two bounds + /// advance without ever moving backwards and the whole sweep is linear in the number + /// of peaks plus the total window occupancy. The per-window slice is summed directly + /// rather than differenced out of a prefix sum, so the result is bit-identical to the + /// training path's . + /// + /// + public static void ComputeNeighborWindow( + ReadOnlySpan mz, + ReadOnlySpan intensity, + double low, + double high, + Span destination) + { + int n = mz.Length; + int lowIdx = 0, highIdx = 0; + for (int i = 0; i < n; i++) + { + double lowBound = mz[i] + low; + double highBound = mz[i] + high; + + while (lowIdx < n && mz[lowIdx] <= lowBound) lowIdx++; + if (highIdx < lowIdx) highIdx = lowIdx; + while (highIdx < n && mz[highIdx] <= highBound) highIdx++; + + double sum = 0.0; + for (int j = lowIdx; j < highIdx; j++) sum += intensity[j]; + destination[i] = sum; + } + } +} diff --git a/dotnet/MARS.Core/PeptideMass.cs b/dotnet/MARS.Core/PeptideMass.cs new file mode 100644 index 0000000..79359ad --- /dev/null +++ b/dotnet/MARS.Core/PeptideMass.cs @@ -0,0 +1,190 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Monoisotopic peptide fragment masses. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace MARS.Core; + +/// +/// Theoretical fragment m/z for peptide backbone ions. +/// +/// This matters for BiblioSpec libraries specifically. A blib stores the OBSERVED m/z of +/// each reference peak, which carries whatever miscalibration the reference run had -- the +/// very thing MARS exists to remove. Using it as ground truth would teach the model to +/// reproduce that error, so b and y fragment m/z are recomputed from the sequence instead. +/// +/// +public static class PeptideMass +{ + public const double Proton = 1.007276466; + + public const double Water = 18.0105646863; + + public const double Ammonia = 17.0265491015; + + public const double CarbonMonoxide = 27.9949146221; + + private static readonly double[] ResidueMass = BuildResidueTable(); + + /// Monoisotopic residue mass, or NaN for an unknown symbol. + public static double Residue(char aminoAcid) + { + int index = aminoAcid - 'A'; + return index >= 0 && index < ResidueMass.Length ? ResidueMass[index] : double.NaN; + } + + public static bool IsKnownResidue(char aminoAcid) => !double.IsNaN(Residue(aminoAcid)); + + /// + /// m/z of a backbone fragment. + /// + /// Unmodified sequence, one letter per residue. + /// 'b', 'y', 'a', 'c', 'x' or 'z'. + /// Residue count in the fragment, 1-based. + /// Fragment charge, at least 1. + /// + /// Mass deltas by 1-based residue position across the whole peptide. Only the deltas + /// falling inside the fragment are applied. + /// + /// The m/z, or NaN when the fragment cannot be computed. + public static double FragmentMz( + string sequence, + char ionType, + int ionNumber, + int charge, + IReadOnlyList<(int Position, double Mass)>? modifications = null) + { + if (charge < 1 || ionNumber < 1 || ionNumber > sequence.Length) return double.NaN; + + bool nTerminal = ionType is 'a' or 'b' or 'c'; + bool cTerminal = ionType is 'x' or 'y' or 'z'; + if (!nTerminal && !cTerminal) return double.NaN; + + int from, to; // 0-based, [from, to) + if (nTerminal) + { + from = 0; + to = ionNumber; + } + else + { + from = sequence.Length - ionNumber; + to = sequence.Length; + } + + double residues = 0; + for (int i = from; i < to; i++) + { + double mass = Residue(sequence[i]); + if (double.IsNaN(mass)) return double.NaN; + residues += mass; + } + + if (modifications is not null) + { + foreach ((int position, double mass) in modifications) + { + int index = position - 1; + if (index >= from && index < to) residues += mass; + } + } + + // b is the residue sum; y adds the C-terminal water. The rest are offsets from those. + double neutral = ionType switch + { + 'b' => residues, + 'a' => residues - CarbonMonoxide, + 'c' => residues + Ammonia, + 'y' => residues + Water, + 'x' => residues + Water + CarbonMonoxide - (2 * 1.00782503207), + 'z' => residues + Water - Ammonia, + _ => double.NaN, + }; + + if (double.IsNaN(neutral)) return double.NaN; + return (neutral + (charge * Proton)) / charge; + } + + /// + /// Splits a modified sequence into its bare residues and the mass delta at each + /// 1-based position, and reports how many modifications it could not weigh. + /// + /// + /// Only a numeric body carries its own mass: M[+15.9949] is a delta this can use, + /// where C[Carbamidomethyl (C)] and M(unimod:35) are names that need a + /// table this does not have. Those are counted rather than ignored. A dropped + /// modification does not produce a missing answer, it produces a confident wrong one - + /// the residue keeps its unmodified mass and every fragment past it is off by the delta - + /// so a caller computing theoretical m/z has to know the difference. + /// + public static (string Stripped, List<(int Position, double Mass)> Modifications, int Unweighed) + SplitModifiedSequence(string sequence) + { + var stripped = new StringBuilder(sequence.Length); + var modifications = new List<(int, double)>(); + var unweighed = 0; + + for (var i = 0; i < sequence.Length; i++) + { + char c = sequence[i]; + + if (c is '[' or '(') + { + char closing = c == '[' ? ']' : ')'; + int end = sequence.IndexOf(closing, i + 1); + if (end < 0) break; + + string body = sequence[(i + 1)..end]; + if (double.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out double delta)) + { + if (stripped.Length > 0) modifications.Add((stripped.Length, delta)); + } + else + { + unweighed++; + } + + i = end; + continue; + } + + if (char.IsLetter(c)) stripped.Append(char.ToUpperInvariant(c)); + } + + return (stripped.ToString(), modifications, unweighed); + } + + private static double[] BuildResidueTable() + { + var table = new double[26]; + for (var i = 0; i < table.Length; i++) table[i] = double.NaN; + + table['G' - 'A'] = 57.02146372; + table['A' - 'A'] = 71.03711378; + table['S' - 'A'] = 87.03202840; + table['P' - 'A'] = 97.05276384; + table['V' - 'A'] = 99.06841390; + table['T' - 'A'] = 101.04767846; + table['C' - 'A'] = 103.00918447; + table['L' - 'A'] = 113.08406396; + table['I' - 'A'] = 113.08406396; + table['J' - 'A'] = 113.08406396; + table['N' - 'A'] = 114.04292744; + table['D' - 'A'] = 115.02694302; + table['Q' - 'A'] = 128.05857750; + table['K' - 'A'] = 128.09496301; + table['E' - 'A'] = 129.04259308; + table['M' - 'A'] = 131.04048508; + table['H' - 'A'] = 137.05891186; + table['F' - 'A'] = 147.06841390; + table['R' - 'A'] = 156.10111102; + table['Y' - 'A'] = 163.06332852; + table['W' - 'A'] = 186.07931294; + table['U' - 'A'] = 150.95363000; + table['O' - 'A'] = 237.14772677; + return table; + } +} diff --git a/dotnet/MARS.Core/SpectralLibrary.cs b/dotnet/MARS.Core/SpectralLibrary.cs new file mode 100644 index 0000000..1ea3640 --- /dev/null +++ b/dotnet/MARS.Core/SpectralLibrary.cs @@ -0,0 +1,229 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from the MARS Python implementation (mars/library.py). + +using System; +using System.Collections.Generic; + +namespace MARS.Core; + +/// +/// Column-oriented spectral library. +/// +/// One "entry" is a precursor: a modified sequence plus a charge state. Each entry +/// owns a contiguous run of fragments in the fragment arrays, delimited by +/// FragmentStart[i] .. FragmentStart[i + 1]. +/// +/// +/// The struct-of-arrays layout matters: a Skyline PRISM report for a plate of Astral +/// runs is tens of millions of fragment rows, and one managed object per fragment +/// would spend more memory on object headers than on data. +/// +/// +public sealed class SpectralLibrary +{ + /// Precursor m/z, in Th. + public required double[] PrecursorMz { get; init; } + + public required int[] PrecursorCharge { get; init; } + + /// Start of the RT window used for matching, in minutes; NaN when unknown. + public required double[] RtStart { get; init; } + + /// End of the RT window used for matching, in minutes; NaN when unknown. + public required double[] RtEnd { get; init; } + + /// Length EntryCount + 1. Fragment range for entry i is [i], [i + 1]. + public required int[] FragmentStart { get; init; } + + /// Only populated when per-match reporting is requested; null otherwise. + public string[]? ModifiedSequence { get; init; } + + /// + /// Peptide identity per entry, as a dense index. Entries whose modified sequence is the + /// same share a value, so grouping rows by this groups them by peptide across charge + /// states and across the duplicate entries a report can produce for one precursor. + /// + /// + /// An int rather than the sequence itself, because the point is only whether two entries + /// are the same peptide, and a plate-scale library has millions of entries. This is what + /// cross-validation folds are assigned over: fragments of one peptide recur across many + /// spectra with identical theoretical m/z, so splitting them across a train/test boundary + /// lets the model memorize that m/z rather than learn the instrument's error. + /// + public required int[] PeptideGroup { get; init; } + + /// Theoretical (library) fragment m/z. This is the calibration ground truth. + public required double[] FragmentMz { get; init; } + + /// Library relative intensity or peak area. Carried for reporting only. + public required float[] FragmentIntensity { get; init; } + + /// ASCII code of the ion type ('y', 'b', ...), or '?' when unannotated. + public required byte[] FragmentIonType { get; init; } + + public required short[] FragmentIonNumber { get; init; } + + public required byte[] FragmentCharge { get; init; } + + public int EntryCount => PrecursorMz.Length; + + public int FragmentCount => FragmentMz.Length; + + /// + /// Entry indices ordered by ascending precursor m/z, with the entry index as a + /// tiebreaker so the order is total and reproducible. + /// + public int[] OrderByPrecursorMz() + { + int n = EntryCount; + var order = new int[n]; + for (int i = 0; i < n; i++) order[i] = i; + double[] mz = PrecursorMz; + Array.Sort(order, (a, b) => + { + int c = mz[a].CompareTo(mz[b]); + return c != 0 ? c : a.CompareTo(b); + }); + return order; + } +} + +/// Growable builder for . +public sealed class SpectralLibraryBuilder +{ + private readonly List _precursorMz = new(); + private readonly List _precursorCharge = new(); + private readonly List _rtStart = new(); + private readonly List _rtEnd = new(); + private readonly List _fragmentStart = new(); + private readonly List? _modifiedSequence; + private readonly List _peptideGroup = new(); + private readonly Dictionary _peptideGroupIds = new(StringComparer.Ordinal); + private int _nextPeptideGroupId; + + private readonly List _fragmentMz = new(); + private readonly List _fragmentIntensity = new(); + private readonly List _fragmentIonType = new(); + private readonly List _fragmentIonNumber = new(); + private readonly List _fragmentCharge = new(); + + private readonly HashSet? _fragmentKeys; + + public SpectralLibraryBuilder(bool keepSequences = false, bool dedupeFragments = true) + { + _modifiedSequence = keepSequences ? new List() : null; + if (dedupeFragments) _fragmentKeys = new HashSet(); + } + + public int EntryCount => _precursorMz.Count; + + public int FragmentCount => _fragmentMz.Count; + + /// Opens a new entry. Subsequent AddFragment calls belong to it. + public int BeginEntry(string modifiedSequence, int charge, double precursorMz, double rtStart, double rtEnd) + { + _fragmentStart.Add(_fragmentMz.Count); + _peptideGroup.Add(GroupIdFor(modifiedSequence)); + _precursorMz.Add(precursorMz); + _precursorCharge.Add(charge); + _rtStart.Add(rtStart); + _rtEnd.Add(rtEnd); + _modifiedSequence?.Add(modifiedSequence); + _fragmentKeys?.Clear(); + return _precursorMz.Count - 1; + } + + /// + /// Adds a fragment to the open entry. When deduplication is on, a fragment whose + /// (m/z, charge, ion type, ion number) already appears in this entry is dropped: + /// Skyline reports repeat every transition once per replicate and the theoretical + /// m/z is identical across replicates, so the copies are exact duplicates. + /// + public bool AddFragment(double mz, double intensity, char ionType, int ionNumber, int charge) + { + if (_fragmentKeys is not null) + { + // Quantize m/z to 1e-6 Th so bit-level noise cannot defeat the key. + long key = (long)Math.Round(mz * 1000000.0); + key = (key * 31) + charge; + key = (key * 31) + ionNumber; + key = (key * 31) + ionType; + if (!_fragmentKeys.Add(key)) return false; + } + + _fragmentMz.Add(mz); + _fragmentIntensity.Add((float)intensity); + _fragmentIonType.Add((byte)ionType); + _fragmentIonNumber.Add((short)Math.Clamp(ionNumber, 0, short.MaxValue)); + _fragmentCharge.Add((byte)Math.Clamp(charge, 0, 255)); + return true; + } + + /// + /// Drops the open entry if it collected no fragments. Mirrors the Python loaders. + /// + /// + /// Every per-entry array has to shed the entry, or they index different entries from each + /// other for the rest of the load. The peptide group is the one that fails quietly: it is + /// what keeps a peptide's fragments inside one cross-validation fold, so a misaligned group + /// array reintroduces the leak the grouped split exists to prevent, and the only symptom is + /// a held-out accuracy that is better than the truth. + /// + public void EndEntry() + { + int last = _precursorMz.Count - 1; + if (last < 0) return; + if (_fragmentStart[last] != _fragmentMz.Count) return; + + _fragmentStart.RemoveAt(last); + _peptideGroup.RemoveAt(last); + _precursorMz.RemoveAt(last); + _precursorCharge.RemoveAt(last); + _rtStart.RemoveAt(last); + _rtEnd.RemoveAt(last); + _modifiedSequence?.RemoveAt(last); + } + + /// + /// Dense id for a peptide sequence. A reader with no sequence to give - some formats + /// carry none - gets one group per entry, which degrades cross-validation to + /// precursor-level grouping rather than silently pooling unrelated entries together. + /// + private int GroupIdFor(string modifiedSequence) + { + // One shared counter, so an unnamed entry can never be handed an id that a named + // one already owns. + if (string.IsNullOrEmpty(modifiedSequence)) return _nextPeptideGroupId++; + + if (!_peptideGroupIds.TryGetValue(modifiedSequence, out int id)) + { + id = _nextPeptideGroupId++; + _peptideGroupIds[modifiedSequence] = id; + } + + return id; + } + + public SpectralLibrary Build() + { + var starts = new int[_fragmentStart.Count + 1]; + _fragmentStart.CopyTo(starts); + starts[_fragmentStart.Count] = _fragmentMz.Count; + + return new SpectralLibrary + { + PrecursorMz = _precursorMz.ToArray(), + PrecursorCharge = _precursorCharge.ToArray(), + RtStart = _rtStart.ToArray(), + RtEnd = _rtEnd.ToArray(), + FragmentStart = starts, + ModifiedSequence = _modifiedSequence?.ToArray(), + PeptideGroup = _peptideGroup.ToArray(), + FragmentMz = _fragmentMz.ToArray(), + FragmentIntensity = _fragmentIntensity.ToArray(), + FragmentIonType = _fragmentIonType.ToArray(), + FragmentIonNumber = _fragmentIonNumber.ToArray(), + FragmentCharge = _fragmentCharge.ToArray(), + }; + } +} diff --git a/dotnet/MARS.Core/Spectrum.cs b/dotnet/MARS.Core/Spectrum.cs new file mode 100644 index 0000000..7cced2f --- /dev/null +++ b/dotnet/MARS.Core/Spectrum.cs @@ -0,0 +1,78 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from the MARS Python implementation (mars/mzml.py). + +using System; + +namespace MARS.Core; + +/// +/// One spectrum handed to MARS by the reader. The arrays are owned by the reader and +/// are only valid for the duration of the callback that receives them. +/// +public sealed class SpectrumRecord +{ + /// Scan number parsed out of the nativeID, or the spectrum index as a fallback. + public int ScanNumber; + + /// Zero-based position in the spectrum list. + public int Index; + + /// mzML spectrum id, e.g. "controllerType=0 controllerNumber=1 scan=42". + public string Id = string.Empty; + + public int MsLevel; + + /// + /// The spectrum's instrumentConfigurationRef, or null when it inherits the run default. + /// On a hybrid instrument this is what says which analyzer recorded the scan. + /// + public string? InstrumentConfigurationRef; + + /// + /// Thermo's scan filter (MS:1000512), when present. A fallback for identifying the + /// analyzer on files whose instrument configuration does not settle it. + /// + public string? FilterString; + + /// Scan start time, in minutes. + public double RetentionTime; + + /// Isolation window lower bound (target - lower offset), in Th. + public double PrecursorMzLow; + + /// Isolation window upper bound (target + upper offset), in Th. + public double PrecursorMzHigh; + + /// Isolation window target m/z, in Th. This is the precursor_mz feature. + public double PrecursorMzCenter; + + /// + /// Sum of the decoded intensity array. This is what the Python matcher calls "tic" + /// and what the log_tic and tic_injection_time features are computed from. + /// + public double SummedIntensity; + + /// Value of the MS:1000285 total ion current cvParam, or 0 when absent. + public double ReportedTic; + + /// Ion injection time in SECONDS (the mzML cvParam is in milliseconds), or null. + public double? InjectionTime; + + /// Run start time as a Unix timestamp in seconds, or null when unparseable. + public double? AcquisitionStartTime; + + /// Acquisition start + RT, in seconds. Normalization happens later. + public double AbsoluteTime; + + public double[] MzArray = Array.Empty(); + + public double[] IntensityArray = Array.Empty(); + + public int PeakCount; + + public double IsolationWindowWidth => PrecursorMzHigh - PrecursorMzLow; + + public ReadOnlySpan Mz => MzArray.AsSpan(0, PeakCount); + + public ReadOnlySpan Intensity => IntensityArray.AsSpan(0, PeakCount); +} diff --git a/dotnet/MARS.Core/SpectrumCorrector.cs b/dotnet/MARS.Core/SpectrumCorrector.cs new file mode 100644 index 0000000..1da198c --- /dev/null +++ b/dotnet/MARS.Core/SpectrumCorrector.cs @@ -0,0 +1,269 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Inference side of the calibration, ported from MzCalibrator.create_calibration_function +// in mars/calibration.py. + +using System; + +namespace MARS.Core; + +/// What to do when a per-peak correction would reorder adjacent peaks. +public enum MonotonicityPolicy +{ + /// + /// Nudge the offending peak up to the next representable double above its predecessor. + /// Keeps the array strictly ascending with the smallest possible perturbation. + /// + ClampAscending, + + /// Leave the whole spectrum uncorrected and count it. + RevertSpectrum, + + /// Write the corrected values as-is. Produces an unsorted m/z array. + Allow, +} + +public sealed class CorrectionOptions +{ + /// Do not correct spectra whose isolation window is wider than this, in Th. + public double? MaxIsolationWindowWidth { get; set; } + + public MonotonicityPolicy Monotonicity { get; set; } = MonotonicityPolicy.ClampAscending; + + /// + /// Reproduce two inconsistencies in the Python implementation, for A/B comparison + /// against its output. Both change which side of a tree split an inference row lands on. + /// + /// log_tic and tic_injection_time are computed from the mzML total ion current + /// cvParam at correction time, but from the summed intensity array at training time. + /// The two differ on Thermo centroided data. + /// absolute_time is fed in as a raw Unix timestamp at correction time, but is + /// re-based to the earliest acquisition before training, so every inference row sits + /// far above the largest value the model ever saw. + /// + /// + public bool PythonCompatibility { get; set; } +} + +/// Per-thread scratch for correcting a spectrum. Reused across spectra. +public sealed class CorrectionWorkspace +{ + private double[] _features = Array.Empty(); + private double[] _neighbors = Array.Empty(); + private double[] _corrections = Array.Empty(); + private double[] _row = Array.Empty(); + + public double[] Features => _features; + + public double[] Neighbors => _neighbors; + + public double[] Corrections => _corrections; + + public double[] Row => _row; + + public void EnsureCapacity(int peakCount, int featureCount) + { + int needed = peakCount * featureCount; + if (_features.Length < needed) _features = new double[Math.Max(needed, 1024)]; + if (_neighbors.Length < peakCount * MarsFeatures.NeighborWindows.Length) + _neighbors = new double[Math.Max(peakCount * MarsFeatures.NeighborWindows.Length, 1024)]; + if (_corrections.Length < peakCount) _corrections = new double[Math.Max(peakCount, 1024)]; + if (_row.Length < featureCount) _row = new double[featureCount]; + } +} + +public sealed class SpectrumCorrectionResult +{ + public bool Corrected { get; init; } + + public int MonotonicityFixes { get; init; } + + public bool Reverted { get; init; } +} + +/// Applies a trained to the peaks of a spectrum. +public sealed class SpectrumCorrector +{ + private readonly MzCalibrator _calibrator; + private readonly CorrectionOptions _options; + private readonly MarsFeature[] _features; + + public SpectrumCorrector(MzCalibrator calibrator, CorrectionOptions options) + { + _calibrator = calibrator; + _options = options; + _features = calibrator.Features.Features; + } + + public bool ShouldCorrect(SpectrumRecord spectrum) + { + if (spectrum.MsLevel != 2) return false; + if (spectrum.PeakCount == 0) return false; + if (_options.MaxIsolationWindowWidth is double maxWidth && spectrum.IsolationWindowWidth > maxWidth) + return false; + return true; + } + + /// + /// Writes corrected m/z values for every peak into . + /// + public SpectrumCorrectionResult Correct( + SpectrumRecord spectrum, + TemperatureSet? temperatures, + CorrectionWorkspace workspace, + Span destination) + { + int n = spectrum.PeakCount; + ReadOnlySpan mz = spectrum.Mz; + ReadOnlySpan intensity = spectrum.Intensity; + + if (!ShouldCorrect(spectrum)) + { + mz.CopyTo(destination); + return new SpectrumCorrectionResult { Corrected = false }; + } + + int nFeat = _features.Length; + workspace.EnsureCapacity(n, nFeat); + double[] features = workspace.Features; + double[] neighbors = workspace.Neighbors; + + double injectionTime = spectrum.InjectionTime ?? 0.0; + + // Training computes the TIC features from the summed intensity array; do the same + // here so inference rows land on the scale the model was fitted on. + double tic = _options.PythonCompatibility ? spectrum.ReportedTic : spectrum.SummedIntensity; + double logTic = Math.Log10(Math.Max(tic, 1.0)); + double ticInjectionTime = tic * injectionTime; + + double absoluteTime = _options.PythonCompatibility + ? spectrum.AbsoluteTime + : spectrum.AbsoluteTime - _calibrator.AbsoluteTimeOffset; + + double rfa2 = temperatures?.Rfa2 is { } a ? a.TemperatureAt(spectrum.RetentionTime) : 0.0; + double rfc2 = temperatures?.Rfc2 is { } c ? c.TemperatureAt(spectrum.RetentionTime) : 0.0; + if (double.IsNaN(rfa2)) rfa2 = 0.0; + if (double.IsNaN(rfc2)) rfc2 = 0.0; + + bool needNeighbors = _calibrator.Features.NeedsNeighborDensity && injectionTime > 0; + if (needNeighbors) + { + for (int w = 0; w < MarsFeatures.NeighborWindows.Length; w++) + { + (double low, double high) = MarsFeatures.NeighborWindows[w]; + PeakSearch.ComputeNeighborWindow(mz, intensity, low, high, neighbors.AsSpan(w * n, n)); + } + } + + for (int i = 0; i < n; i++) + { + double peakMz = mz[i]; + double peakIntensity = intensity[i]; + double fragmentIons = peakIntensity * injectionTime; + int rowStart = i * nFeat; + + for (int j = 0; j < nFeat; j++) + { + features[rowStart + j] = FeatureValue( + _features[j], peakMz, peakIntensity, fragmentIons, injectionTime, + logTic, ticInjectionTime, absoluteTime, spectrum.PrecursorMzCenter, + rfa2, rfc2, neighbors, needNeighbors, i, n); + } + } + + double[] corrections = workspace.Corrections; + double[] row = workspace.Row; + for (int i = 0; i < n; i++) + { + Array.Copy(features, i * nFeat, row, 0, nFeat); + corrections[i] = _calibrator.PredictDelta(row); + } + + int fixes = 0; + double previous = double.NegativeInfinity; + for (int i = 0; i < n; i++) + { + double corrected = mz[i] - corrections[i]; + if (corrected <= previous) + { + fixes++; + if (_options.Monotonicity == MonotonicityPolicy.RevertSpectrum) + { + mz.CopyTo(destination); + return new SpectrumCorrectionResult + { + Corrected = false, + Reverted = true, + MonotonicityFixes = fixes, + }; + } + + if (_options.Monotonicity == MonotonicityPolicy.ClampAscending) + corrected = Math.BitIncrement(previous); + } + + destination[i] = corrected; + previous = corrected; + } + + return new SpectrumCorrectionResult { Corrected = true, MonotonicityFixes = fixes }; + } + + private static double FeatureValue( + MarsFeature feature, + double peakMz, + double peakIntensity, + double fragmentIons, + double injectionTime, + double logTic, + double ticInjectionTime, + double absoluteTime, + double precursorMz, + double rfa2, + double rfc2, + double[] neighbors, + bool haveNeighbors, + int peakIndex, + int peakCount) + { + switch (feature) + { + case MarsFeature.PrecursorMz: + return precursorMz; + case MarsFeature.FragmentMz: + // Training uses the theoretical library m/z here; correction has only the + // observed value. They differ by at most the matching tolerance. + return peakMz; + case MarsFeature.LogTic: + return logTic; + case MarsFeature.LogIntensity: + return Math.Log10(Math.Max(peakIntensity, 1.0)); + case MarsFeature.AbsoluteTime: + return absoluteTime; + case MarsFeature.InjectionTime: + return injectionTime; + case MarsFeature.TicInjectionTime: + return ticInjectionTime; + case MarsFeature.FragmentIons: + return fragmentIons; + default: + break; + } + + if (feature == MarsFeature.Rfa2Temp) return rfa2; + if (feature == MarsFeature.Rfc2Temp) return rfc2; + + for (int w = 0; w < MarsFeatures.NeighborFeatures.Length; w++) + { + if (MarsFeatures.NeighborFeatures[w] == feature) + return haveNeighbors ? neighbors[(w * peakCount) + peakIndex] * injectionTime : 0.0; + + if (MarsFeatures.RatioFeatures[w] == feature) + { + if (!haveNeighbors || fragmentIons <= 0) return 0.0; + return neighbors[(w * peakCount) + peakIndex] * injectionTime / fragmentIons; + } + } + + return 0.0; + } +} diff --git a/dotnet/MARS.Core/Statistics.cs b/dotnet/MARS.Core/Statistics.cs new file mode 100644 index 0000000..fd64ea6 --- /dev/null +++ b/dotnet/MARS.Core/Statistics.cs @@ -0,0 +1,112 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; + +namespace MARS.Core; + +/// Summary of a mass-error distribution, in Th. +public readonly struct ErrorSummary +{ + public required int Count { get; init; } + + public required double Mean { get; init; } + + public required double Median { get; init; } + + /// Sample standard deviation, matching pandas Series.std (ddof = 1). + public required double StdDev { get; init; } + + /// Mean absolute error. + public required double Mae { get; init; } + + /// Root mean square. + public required double Rms { get; init; } + + /// Median absolute deviation about the median. + public required double Mad { get; init; } +} + +public static class MarsStatistics +{ + public static double Mean(ReadOnlySpan values) + { + if (values.Length == 0) return double.NaN; + double sum = 0; + for (int i = 0; i < values.Length; i++) sum += values[i]; + return sum / values.Length; + } + + /// Sample standard deviation with ddof = 1, matching pandas. + public static double StdDev(ReadOnlySpan values) + { + if (values.Length < 2) return double.NaN; + double mean = Mean(values); + double sum = 0; + for (int i = 0; i < values.Length; i++) + { + double d = values[i] - mean; + sum += d * d; + } + + return Math.Sqrt(sum / (values.Length - 1)); + } + + public static double Rms(ReadOnlySpan values) + { + if (values.Length == 0) return double.NaN; + double sum = 0; + for (int i = 0; i < values.Length; i++) sum += values[i] * values[i]; + return Math.Sqrt(sum / values.Length); + } + + public static double MeanAbsolute(ReadOnlySpan values) + { + if (values.Length == 0) return double.NaN; + double sum = 0; + for (int i = 0; i < values.Length; i++) sum += Math.Abs(values[i]); + return sum / values.Length; + } + + /// + /// Linearly interpolated median, matching numpy. Sorts a copy, so the caller's array + /// is left alone. + /// + public static double Median(ReadOnlySpan values) + { + if (values.Length == 0) return double.NaN; + var copy = values.ToArray(); + Array.Sort(copy); + return MedianOfSorted(copy); + } + + public static double MedianOfSorted(double[] sorted) + { + int n = sorted.Length; + if (n == 0) return double.NaN; + int mid = n / 2; + return (n & 1) == 1 ? sorted[mid] : 0.5 * (sorted[mid - 1] + sorted[mid]); + } + + /// Median absolute deviation about the median. + public static double MedianAbsoluteDeviation(ReadOnlySpan values) + { + if (values.Length == 0) return double.NaN; + var copy = values.ToArray(); + Array.Sort(copy); + double median = MedianOfSorted(copy); + for (int i = 0; i < copy.Length; i++) copy[i] = Math.Abs(copy[i] - median); + Array.Sort(copy); + return MedianOfSorted(copy); + } + + public static ErrorSummary Summarize(ReadOnlySpan values) => new() + { + Count = values.Length, + Mean = Mean(values), + Median = Median(values), + StdDev = StdDev(values), + Mae = MeanAbsolute(values), + Rms = Rms(values), + Mad = MedianAbsoluteDeviation(values), + }; +} diff --git a/dotnet/MARS.Core/TemperatureData.cs b/dotnet/MARS.Core/TemperatureData.cs new file mode 100644 index 0000000..e01e6f0 --- /dev/null +++ b/dotnet/MARS.Core/TemperatureData.cs @@ -0,0 +1,95 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from mars/temperature.py. + +using System; + +namespace MARS.Core; + +/// +/// An RF-generator temperature trace, sampled against chromatographic time. +/// Lookups are nearest-neighbor, as in the Python implementation. +/// +public sealed class TemperatureData +{ + private readonly double[] _timeMinutes; + private readonly double[] _temperature; + + public TemperatureData(double[] timeMinutes, double[] temperature, string source) + { + if (timeMinutes.Length != temperature.Length) + throw new ArgumentException("Time and temperature arrays must be the same length."); + + // Sort by time so lookups can binary search. Array.Sort with a key array is a + // stable-enough total order here because duplicate times carry identical readings. + var order = new int[timeMinutes.Length]; + for (int i = 0; i < order.Length; i++) order[i] = i; + Array.Sort(order, (a, b) => + { + int c = timeMinutes[a].CompareTo(timeMinutes[b]); + return c != 0 ? c : a.CompareTo(b); + }); + + _timeMinutes = new double[order.Length]; + _temperature = new double[order.Length]; + for (int i = 0; i < order.Length; i++) + { + _timeMinutes[i] = timeMinutes[order[i]]; + _temperature[i] = temperature[order[i]]; + } + + Source = source; + } + + public string Source { get; } + + public int Count => _timeMinutes.Length; + + public double MinTemperature + { + get + { + double min = double.PositiveInfinity; + foreach (double t in _temperature) if (t < min) min = t; + return _temperature.Length == 0 ? double.NaN : min; + } + } + + public double MaxTemperature + { + get + { + double max = double.NegativeInfinity; + foreach (double t in _temperature) if (t > max) max = t; + return _temperature.Length == 0 ? double.NaN : max; + } + } + + /// + /// Temperature at a retention time, in degrees C, using the nearest sample. + /// Ties resolve to the earlier sample, matching numpy argmin. + /// Returns NaN when the trace is empty. + /// + public double TemperatureAt(double retentionTimeMinutes) + { + int n = _timeMinutes.Length; + if (n == 0) return double.NaN; + + int idx = PeakSearch.LowerBound(_timeMinutes, retentionTimeMinutes); + if (idx == 0) return _temperature[0]; + if (idx >= n) return _temperature[n - 1]; + + double distPrev = retentionTimeMinutes - _timeMinutes[idx - 1]; + double distCurr = _timeMinutes[idx] - retentionTimeMinutes; + return distPrev <= distCurr ? _temperature[idx - 1] : _temperature[idx]; + } +} + +/// The RF temperature traces available for a single run. +public sealed class TemperatureSet +{ + public TemperatureData? Rfa2 { get; init; } + + public TemperatureData? Rfc2 { get; init; } + + public bool IsEmpty => Rfa2 is null && Rfc2 is null; +} diff --git a/dotnet/MARS.IO/BlibLibraryReader.cs b/dotnet/MARS.IO/BlibLibraryReader.cs new file mode 100644 index 0000000..5fd223a --- /dev/null +++ b/dotnet/MARS.IO/BlibLibraryReader.cs @@ -0,0 +1,335 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from load_blib in mars/library.py, reading BiblioSpec libraries through the +// managed SQLite reader rather than a native provider. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Runtime.InteropServices; +using MARS.Core; +using MARS.IO.Sqlite; + +namespace MARS.IO; + +public static class BlibLibraryReader +{ + private sealed class SpectrumMeta + { + public string PeptideSequence = string.Empty; + public string ModifiedSequence = string.Empty; + public double PrecursorMz; + public int PrecursorCharge; + public double RetentionTime = double.NaN; + public int NumPeaks; + } + + /// + /// Loads a BiblioSpec library. + /// + /// Path to the .blib file. + /// + /// Half-width of the retention time window placed around each entry's library RT. + /// + /// Progress sink. + /// + /// Recompute b and y fragment m/z from the sequence instead of trusting the stored peak + /// m/z. A blib records OBSERVED reference-spectrum m/z, which carries the reference + /// run's own miscalibration, so leaving this on is what makes a blib usable as ground + /// truth at all. + /// + /// + /// Skip peaks the library does not annotate as a fragment ion. An unannotated peak has + /// only its OBSERVED m/z from the reference run, so matching it measures the difference + /// between two runs' calibration errors rather than an absolute mass error, and a + /// library with hundreds of unannotated peaks per spectrum swamps the real fragments. + /// The Python implementation keeps them; MARS does not, by default. + /// + public static SpectralLibrary Load( + string path, + double rtWindowMinutes = 0.083, + Action? log = null, + bool recalculateFragmentMz = true, + bool annotatedPeaksOnly = true) + { + if (!File.Exists(path)) throw new FileNotFoundException("BiblioSpec library not found.", path); + + using var database = new SqliteFile(path); + + SqliteTable refSpectra = database.Table("RefSpectra"); + SqliteTable refSpectraPeaks = database.Table("RefSpectraPeaks"); + + var metaById = new Dictionary(); + int idAt = refSpectra.ColumnIndex("id"); + int peptideAt = refSpectra.ColumnIndex("peptideSeq"); + int precursorMzAt = refSpectra.ColumnIndex("precursorMZ"); + int chargeAt = refSpectra.ColumnIndex("precursorCharge"); + int modSeqAt = refSpectra.ColumnIndex("peptideModSeq"); + int rtAt = refSpectra.ColumnIndex("retentionTime"); + int numPeaksAt = refSpectra.ColumnIndex("numPeaks"); + + foreach (SqliteRow row in database.Scan(refSpectra)) + { + // "id INTEGER PRIMARY KEY" is an alias for the rowid: SQLite stores NULL in the + // record and the real value only exists in the cell's rowid. + long id = idAt >= 0 && !row[idAt].IsNull ? row[idAt].AsInteger() : row.RowId; + metaById[id] = new SpectrumMeta + { + PeptideSequence = peptideAt >= 0 ? row[peptideAt].AsText() : string.Empty, + ModifiedSequence = modSeqAt >= 0 ? row[modSeqAt].AsText() : string.Empty, + PrecursorMz = precursorMzAt >= 0 ? row[precursorMzAt].AsDouble() : double.NaN, + PrecursorCharge = chargeAt >= 0 ? (int)row[chargeAt].AsInteger() : 0, + RetentionTime = rtAt >= 0 && !row[rtAt].IsNull ? row[rtAt].AsDouble() : double.NaN, + NumPeaks = numPeaksAt >= 0 ? (int)row[numPeaksAt].AsInteger() : 0, + }; + } + + log?.Invoke($"blib: {metaById.Count:N0} reference spectra"); + + Dictionary> modificationsById = ReadModifications(database); + Dictionary> annotationsById = + ReadAnnotations(database); + + var builder = new SpectralLibraryBuilder(keepSequences: false, dedupeFragments: false); + int peaksIdAt = refSpectraPeaks.ColumnIndex("RefSpectraID"); + int peakMzAt = refSpectraPeaks.ColumnIndex("peakMZ"); + int peakIntensityAt = refSpectraPeaks.ColumnIndex("peakIntensity"); + + var mzBuffer = new double[512]; + var intensityBuffer = new float[512]; + long entriesWithoutPeaks = 0, recalculated = 0, annotated = 0, unannotatedSkipped = 0; + long entriesWithUnweighedMods = 0; + + foreach (SqliteRow row in database.Scan(refSpectraPeaks)) + { + long id = peaksIdAt >= 0 && !row[peaksIdAt].IsNull ? row[peaksIdAt].AsInteger() : row.RowId; + if (!metaById.TryGetValue(id, out SpectrumMeta? meta)) continue; + + int mzCount = DecodeDoubles(row[peakMzAt].AsBlob(), ref mzBuffer); + int intensityCount = DecodeFloats(row[peakIntensityAt].AsBlob(), ref intensityBuffer); + + if (mzCount == 0 || mzCount != intensityCount) + { + entriesWithoutPeaks++; + continue; + } + + float maxIntensity = 0; + for (var i = 0; i < intensityCount; i++) + { + if (intensityBuffer[i] > maxIntensity) maxIntensity = intensityBuffer[i]; + } + + string sequenceForMass = meta.ModifiedSequence.Length > 0 ? meta.ModifiedSequence : meta.PeptideSequence; + (string stripped, List<(int, double)> parsedMods, int unweighedMods) = + PeptideMass.SplitModifiedSequence(sequenceForMass); + if (stripped.Length == 0) stripped = meta.PeptideSequence; + + // Prefer the Modifications table, which stores exact numeric deltas by position. + bool haveTable = modificationsById.TryGetValue(id, out List<(int, double)>? fromTable) && + fromTable.Count > 0; + List<(int Position, double Mass)>? modifications = haveTable ? fromTable : parsedMods; + + // A modification named rather than weighed - C[Carbamidomethyl] - has no mass here. + // With no Modifications table to fall back on, a recalculated fragment would be + // computed from the unmodified residue and be wrong by the delta, so keep the m/z + // the library recorded instead. + bool canRecalculate = haveTable || unweighedMods == 0; + if (!canRecalculate) entriesWithUnweighedMods++; + + annotationsById.TryGetValue(id, out Dictionary? annotations); + + double rt = meta.RetentionTime; + builder.BeginEntry( + sequenceForMass, + meta.PrecursorCharge, + meta.PrecursorMz, + double.IsNaN(rt) ? double.NaN : rt - rtWindowMinutes, + double.IsNaN(rt) ? double.NaN : rt + rtWindowMinutes); + + for (var i = 0; i < mzCount; i++) + { + double mz = mzBuffer[i]; + double intensity = maxIntensity > 0 ? intensityBuffer[i] / maxIntensity : 0.0; + var ionType = '?'; + var ionNumber = 0; + var charge = 1; + + if (annotations is not null && annotations.TryGetValue(i, out (char IonType, int IonNumber, int Charge) annotation)) + { + ionType = annotation.IonType; + ionNumber = annotation.IonNumber; + charge = annotation.Charge > 0 ? annotation.Charge : 1; + annotated++; + + if (recalculateFragmentMz && canRecalculate && ionType is 'b' or 'y' && ionNumber > 0) + { + double theoretical = PeptideMass.FragmentMz(stripped, ionType, ionNumber, charge, modifications); + if (!double.IsNaN(theoretical) && theoretical > 0) + { + mz = theoretical; + recalculated++; + } + } + } + else if (annotatedPeaksOnly) + { + unannotatedSkipped++; + continue; + } + + builder.AddFragment(mz, intensity, ionType, ionNumber, charge); + } + + builder.EndEntry(); + } + + SpectralLibrary library = builder.Build(); + + log?.Invoke($" {library.EntryCount:N0} precursors, {library.FragmentCount:N0} fragments"); + log?.Invoke($" {annotated:N0} annotated peaks, {recalculated:N0} fragment m/z recomputed from sequence"); + if (unannotatedSkipped > 0) + log?.Invoke($" {unannotatedSkipped:N0} unannotated peaks skipped"); + if (entriesWithoutPeaks > 0) + log?.Invoke($" {entriesWithoutPeaks:N0} spectra skipped for missing or mismatched peak arrays"); + + if (entriesWithUnweighedMods > 0) + { + log?.Invoke( + $" {entriesWithUnweighedMods:N0} entries name a modification without giving its " + + "mass and have no Modifications table; their recorded fragment m/z is used as-is"); + } + + if (annotated == 0) + { + throw new InvalidDataException( + $"'{Path.GetFileName(path)}' carries no peak annotations, so no peak in it can be identified " + + "as a specific fragment ion. Every peak would have to be matched on its OBSERVED reference " + + "m/z, which measures the difference between two runs' calibration errors rather than an " + + "absolute mass error, and would train the model on noise. Use a Skyline PRISM report " + + "(--prism-csv) or a DIA-NN library (--library report-lib.parquet) instead, or rebuild the " + + "library with peak annotations."); + } + + if (recalculated == 0) + { + log?.Invoke(" WARNING: no fragment m/z could be recomputed from sequence, so matching uses the " + + "reference spectra's OBSERVED m/z. Those carry the reference run's own mass error, " + + "which is the thing MARS is trying to remove."); + } + + return library; + } + + private static Dictionary> ReadModifications(SqliteFile database) + { + var result = new Dictionary>(); + if (!database.HasTable("Modifications")) return result; + + SqliteTable table = database.Table("Modifications"); + int idAt = table.ColumnIndex("RefSpectraID"); + int positionAt = table.ColumnIndex("position"); + int massAt = table.ColumnIndex("mass"); + if (idAt < 0 || positionAt < 0 || massAt < 0) return result; + + foreach (SqliteRow row in database.Scan(table)) + { + long id = row[idAt].AsInteger(); + if (!result.TryGetValue(id, out List<(int, double)>? list)) + { + list = new List<(int, double)>(2); + result[id] = list; + } + + list.Add(((int)row[positionAt].AsInteger(), row[massAt].AsDouble())); + } + + return result; + } + + private static Dictionary> ReadAnnotations( + SqliteFile database) + { + var result = new Dictionary>(); + if (!database.HasTable("RefSpectraPeakAnnotations")) return result; + + SqliteTable table = database.Table("RefSpectraPeakAnnotations"); + int idAt = table.ColumnIndex("RefSpectraID"); + int peakIndexAt = table.ColumnIndex("peakIndex"); + int nameAt = table.ColumnIndex("name"); + int chargeAt = table.ColumnIndex("charge"); + if (idAt < 0 || peakIndexAt < 0 || nameAt < 0) return result; + + foreach (SqliteRow row in database.Scan(table)) + { + long id = row[idAt].AsInteger(); + var peakIndex = (int)row[peakIndexAt].AsInteger(); + string name = row[nameAt].AsText(); + int charge = chargeAt >= 0 ? (int)row[chargeAt].AsInteger() : 1; + + (char ionType, int ionNumber) = PrismCsvLibraryReader.ParseFragmentIon(name); + + if (!result.TryGetValue(id, out Dictionary? peaks)) + { + peaks = new Dictionary(); + result[id] = peaks; + } + + // A peak can carry several annotations; the first wins, as in the Python loader. + peaks.TryAdd(peakIndex, (ionType, ionNumber, charge)); + } + + return result; + } + + private static int DecodeDoubles(ReadOnlySpan blob, ref double[] destination) + { + byte[] raw = Decompress(blob, out int length); + int count = length / 8; + if (destination.Length < count) destination = new double[Math.Max(count, 512)]; + MemoryMarshal.Cast(raw.AsSpan(0, count * 8)).CopyTo(destination.AsSpan(0, count)); + return count; + } + + private static int DecodeFloats(ReadOnlySpan blob, ref float[] destination) + { + byte[] raw = Decompress(blob, out int length); + int count = length / 4; + if (destination.Length < count) destination = new float[Math.Max(count, 512)]; + MemoryMarshal.Cast(raw.AsSpan(0, count * 4)).CopyTo(destination.AsSpan(0, count)); + return count; + } + + /// + /// BiblioSpec compresses a peak blob only when that makes it smaller, so a blob may be + /// either zlib or raw little-endian values. + /// + private static byte[] Decompress(ReadOnlySpan blob, out int length) + { + if (blob.Length == 0) + { + length = 0; + return Array.Empty(); + } + + if (blob.Length >= 2 && blob[0] == 0x78) + { + try + { + using var input = new MemoryStream(blob.ToArray(), writable: false); + using var zlib = new ZLibStream(input, CompressionMode.Decompress); + using var output = new MemoryStream(blob.Length * 4); + zlib.CopyTo(output); + length = (int)output.Length; + return output.GetBuffer(); + } + catch (InvalidDataException) + { + // Not actually compressed; fall through to the raw path. + } + } + + length = blob.Length; + return blob.ToArray(); + } +} diff --git a/dotnet/MARS.IO/CsvReader.cs b/dotnet/MARS.IO/CsvReader.cs new file mode 100644 index 0000000..2fab3da --- /dev/null +++ b/dotnet/MARS.IO/CsvReader.cs @@ -0,0 +1,194 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Streaming RFC 4180 CSV reader. +// +// A Skyline PRISM report for a plate of Astral runs is tens of gigabytes and tens of +// millions of rows, so the reader never materializes the file, never allocates a string +// per cell the caller does not ask for, and reuses its row buffer. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace MARS.IO; + +public sealed class CsvReader : IDisposable +{ + private readonly TextReader _reader; + private readonly bool _ownsReader; + private readonly StringBuilder _cell = new(64); + private readonly List _fields = new(); + private string[] _header = Array.Empty(); + private readonly Dictionary _headerIndex = new(StringComparer.Ordinal); + + // Characters are pulled a block at a time and indexed directly. A TextReader.Read() + // per character is a virtual call per character, which on a plate-scale report means + // tens of billions of them. + private readonly char[] _buffer = new char[1 << 16]; + private int _bufferLength; + private int _bufferAt; + + public CsvReader(string path) + : this(new StreamReader(path, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1 << 20), true) + { + } + + public CsvReader(TextReader reader, bool ownsReader = false) + { + _reader = reader; + _ownsReader = ownsReader; + } + + private int NextChar() + { + if (_bufferAt >= _bufferLength) + { + _bufferLength = _reader.Read(_buffer, 0, _buffer.Length); + _bufferAt = 0; + if (_bufferLength <= 0) return -1; + } + + return _buffer[_bufferAt++]; + } + + private int PeekChar() + { + if (_bufferAt >= _bufferLength) + { + _bufferLength = _reader.Read(_buffer, 0, _buffer.Length); + _bufferAt = 0; + if (_bufferLength <= 0) return -1; + } + + return _buffer[_bufferAt]; + } + + public IReadOnlyList Header => _header; + + public long RowNumber { get; private set; } + + /// Reads the header row and builds the column lookup. + public bool ReadHeader() + { + if (!ReadRow()) return false; + _header = _fields.ToArray(); + _headerIndex.Clear(); + for (int i = 0; i < _header.Length; i++) _headerIndex[_header[i]] = i; + return true; + } + + public bool HasColumn(string name) => _headerIndex.ContainsKey(name); + + /// Column index, or -1 when the column is absent. + public int ColumnIndex(string name) => _headerIndex.TryGetValue(name, out int index) ? index : -1; + + public IReadOnlyList RequireColumns(params string[] names) + { + var missing = new List(); + foreach (string name in names) + { + if (!HasColumn(name)) missing.Add(name); + } + + return missing; + } + + /// Reads the next row. Fields are valid until the following call. + public bool ReadRow() + { + _fields.Clear(); + _cell.Clear(); + + int c = NextChar(); + if (c < 0) return false; + + var inQuotes = false; + while (true) + { + if (c < 0) + { + _fields.Add(_cell.ToString()); + break; + } + + var ch = (char)c; + + if (inQuotes) + { + if (ch == '"') + { + if (PeekChar() == '"') + { + NextChar(); + _cell.Append('"'); + } + else + { + inQuotes = false; + } + } + else + { + _cell.Append(ch); + } + } + else if (ch == '"' && _cell.Length == 0) + { + inQuotes = true; + } + else if (ch == ',') + { + _fields.Add(_cell.ToString()); + _cell.Clear(); + } + else if (ch == '\n') + { + _fields.Add(_cell.ToString()); + break; + } + else if (ch == '\r') + { + if (PeekChar() == '\n') NextChar(); + _fields.Add(_cell.ToString()); + break; + } + else + { + _cell.Append(ch); + } + + c = NextChar(); + } + + RowNumber++; + return true; + } + + public int FieldCount => _fields.Count; + + public string Field(int index) => index >= 0 && index < _fields.Count ? _fields[index] : string.Empty; + + public string Field(string name) => Field(ColumnIndex(name)); + + public double DoubleField(int index) + { + string text = Field(index); + return double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out double value) + ? value + : double.NaN; + } + + public int IntField(int index, int fallback = 0) + { + string text = Field(index); + return int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out int value) + ? value + : fallback; + } + + public void Dispose() + { + if (_ownsReader) _reader.Dispose(); + } +} diff --git a/dotnet/MARS.IO/DiannParquetLibraryReader.cs b/dotnet/MARS.IO/DiannParquetLibraryReader.cs new file mode 100644 index 0000000..0fa89be --- /dev/null +++ b/dotnet/MARS.IO/DiannParquetLibraryReader.cs @@ -0,0 +1,304 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from load_diann_library in mars/library.py. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using MARS.Core; +using Parquet; +using Parquet.Data; +using Parquet.Schema; + +namespace MARS.IO; + +public static class DiannParquetLibraryReader +{ + public const string PrecursorIdColumn = "Precursor.Id"; + public const string ModifiedSequenceColumn = "Modified.Sequence"; + public const string StrippedSequenceColumn = "Stripped.Sequence"; + public const string PrecursorChargeColumn = "Precursor.Charge"; + public const string PrecursorMzColumn = "Precursor.Mz"; + public const string ProductMzColumn = "Product.Mz"; + public const string RelativeIntensityColumn = "Relative.Intensity"; + public const string FragmentTypeColumn = "Fragment.Type"; + public const string FragmentChargeColumn = "Fragment.Charge"; + public const string FragmentSeriesNumberColumn = "Fragment.Series.Number"; + public const string RunColumn = "Run"; + public const string RtStartColumn = "RT.Start"; + public const string RtStopColumn = "RT.Stop"; + + /// + /// Loads a DIA-NN spectral library, taking per-run retention time windows from the + /// companion report. + /// + /// report-lib.parquet, holding the fragments. + /// + /// report.parquet, holding RT.Start and RT.Stop. When null, a report.parquet beside the + /// library is used. + /// + /// mzML files being processed; other runs' RT windows are ignored. + public static SpectralLibrary Load( + string libraryPath, + string? reportPath, + IReadOnlyList runNames, + Action? log = null) + { + if (!File.Exists(libraryPath)) + throw new FileNotFoundException("DIA-NN library parquet not found.", libraryPath); + + string report = reportPath ?? Path.Combine( + Path.GetDirectoryName(Path.GetFullPath(libraryPath)) ?? ".", "report.parquet"); + + if (!File.Exists(report)) + { + throw new FileNotFoundException( + $"DIA-NN report.parquet not found next to {Path.GetFileName(libraryPath)}. It supplies the " + + "RT.Start and RT.Stop windows MARS matches within; pass it with --diann-report.", + report); + } + + Dictionary rtWindows = ReadRtWindows(report, runNames, log); + + var builder = new SpectralLibraryBuilder(keepSequences: false, dedupeFragments: true); + long fragmentRows = 0, withRtWindow = 0; + + string currentPrecursor = string.Empty; + var haveEntry = false; + + foreach (LibraryRow row in ReadLibraryRows(libraryPath, log)) + { + fragmentRows++; + + if (!haveEntry || !string.Equals(row.PrecursorId, currentPrecursor, StringComparison.Ordinal)) + { + if (haveEntry) builder.EndEntry(); + + double start = double.NaN, stop = double.NaN; + if (rtWindows.TryGetValue(row.PrecursorId, out (double Start, double Stop) window)) + { + start = window.Start; + stop = window.Stop; + withRtWindow++; + } + + builder.BeginEntry(row.ModifiedSequence, row.PrecursorCharge, row.PrecursorMz, start, stop); + currentPrecursor = row.PrecursorId; + haveEntry = true; + } + + if (double.IsNaN(row.ProductMz) || row.ProductMz <= 0) continue; + + char ionType = row.FragmentType.Length > 0 ? char.ToLowerInvariant(row.FragmentType[0]) : '?'; + builder.AddFragment(row.ProductMz, row.RelativeIntensity, ionType, row.FragmentSeriesNumber, row.FragmentCharge); + } + + if (haveEntry) builder.EndEntry(); + + SpectralLibrary library = builder.Build(); + log?.Invoke($"DIA-NN library: {fragmentRows:N0} fragment rows, {library.EntryCount:N0} precursors, " + + $"{library.FragmentCount:N0} fragments"); + log?.Invoke($" {withRtWindow:N0} precursors carry an RT window from the report"); + + if (withRtWindow == 0) + { + log?.Invoke(" WARNING: no precursor matched a report RT window. Matching will consider every " + + "library precursor at every retention time, which is slow and adds false matches."); + } + + return library; + } + + private readonly struct LibraryRow + { + public LibraryRow( + string precursorId, string modifiedSequence, int precursorCharge, double precursorMz, + double productMz, double relativeIntensity, string fragmentType, int fragmentCharge, + int fragmentSeriesNumber) + { + PrecursorId = precursorId; + ModifiedSequence = modifiedSequence; + PrecursorCharge = precursorCharge; + PrecursorMz = precursorMz; + ProductMz = productMz; + RelativeIntensity = relativeIntensity; + FragmentType = fragmentType; + FragmentCharge = fragmentCharge; + FragmentSeriesNumber = fragmentSeriesNumber; + } + + public string PrecursorId { get; } + + public string ModifiedSequence { get; } + + public int PrecursorCharge { get; } + + public double PrecursorMz { get; } + + public double ProductMz { get; } + + public double RelativeIntensity { get; } + + public string FragmentType { get; } + + public int FragmentCharge { get; } + + public int FragmentSeriesNumber { get; } + } + + private static IEnumerable ReadLibraryRows(string path, Action? log) + { + using Stream stream = File.OpenRead(path); + using ParquetReader reader = ParquetReader.CreateAsync(stream).GetAwaiter().GetResult(); + + DataField[] fields = reader.Schema.GetDataFields(); + RequireFields(path, fields, PrecursorIdColumn, ModifiedSequenceColumn, PrecursorChargeColumn, + PrecursorMzColumn, ProductMzColumn, RelativeIntensityColumn, FragmentTypeColumn, + FragmentChargeColumn, FragmentSeriesNumberColumn); + + for (var group = 0; group < reader.RowGroupCount; group++) + { + using ParquetRowGroupReader groupReader = reader.OpenRowGroupReader(group); + + string[] precursorId = ReadStrings(groupReader, fields, PrecursorIdColumn); + string[] modifiedSequence = ReadStrings(groupReader, fields, ModifiedSequenceColumn); + double[] precursorCharge = ReadDoubles(groupReader, fields, PrecursorChargeColumn); + double[] precursorMz = ReadDoubles(groupReader, fields, PrecursorMzColumn); + double[] productMz = ReadDoubles(groupReader, fields, ProductMzColumn); + double[] relativeIntensity = ReadDoubles(groupReader, fields, RelativeIntensityColumn); + string[] fragmentType = ReadStrings(groupReader, fields, FragmentTypeColumn); + double[] fragmentCharge = ReadDoubles(groupReader, fields, FragmentChargeColumn); + double[] seriesNumber = ReadDoubles(groupReader, fields, FragmentSeriesNumberColumn); + + int rows = precursorId.Length; + for (var i = 0; i < rows; i++) + { + yield return new LibraryRow( + precursorId[i] ?? string.Empty, + modifiedSequence.Length > i ? modifiedSequence[i] ?? string.Empty : string.Empty, + (int)precursorCharge[i], + precursorMz[i], + productMz[i], + relativeIntensity[i], + fragmentType.Length > i ? fragmentType[i] ?? string.Empty : string.Empty, + (int)fragmentCharge[i], + (int)seriesNumber[i]); + } + } + } + + private static Dictionary ReadRtWindows( + string path, IReadOnlyList runNames, Action? log) + { + var windows = new Dictionary(StringComparer.Ordinal); + var filter = new RunNameFilter(runNames); + + using Stream stream = File.OpenRead(path); + using ParquetReader reader = ParquetReader.CreateAsync(stream).GetAwaiter().GetResult(); + + DataField[] fields = reader.Schema.GetDataFields(); + RequireFields(path, fields, PrecursorIdColumn, RunColumn, RtStartColumn, RtStopColumn); + + long rows = 0, kept = 0; + for (var group = 0; group < reader.RowGroupCount; group++) + { + using ParquetRowGroupReader groupReader = reader.OpenRowGroupReader(group); + + string[] precursorId = ReadStrings(groupReader, fields, PrecursorIdColumn); + string[] run = ReadStrings(groupReader, fields, RunColumn); + double[] start = ReadDoubles(groupReader, fields, RtStartColumn); + double[] stop = ReadDoubles(groupReader, fields, RtStopColumn); + + for (var i = 0; i < precursorId.Length; i++) + { + rows++; + if (filter.Active && !filter.Matches(run.Length > i ? run[i] ?? string.Empty : string.Empty)) + continue; + if (double.IsNaN(start[i]) || double.IsNaN(stop[i])) continue; + + string id = precursorId[i] ?? string.Empty; + kept++; + + // Several runs can identify the same precursor; widen to cover them all, so + // a spectrum from any of them still falls inside the window. + if (windows.TryGetValue(id, out (double Start, double Stop) existing)) + { + windows[id] = (Math.Min(existing.Start, start[i]), Math.Max(existing.Stop, stop[i])); + } + else + { + windows[id] = (start[i], stop[i]); + } + } + } + + log?.Invoke($"DIA-NN report: {rows:N0} identifications, {kept:N0} kept, " + + $"{windows.Count:N0} precursors with RT windows"); + return windows; + } + + private static void RequireFields(string path, DataField[] fields, params string[] required) + { + var present = new HashSet(fields.Select(f => f.Name), StringComparer.Ordinal); + var missing = required.Where(name => !present.Contains(name)).ToList(); + if (missing.Count == 0) return; + + // A report.parquet handed in where a report-lib.parquet belongs is the usual mistake. + if (present.Contains(RtStartColumn) && present.Contains(RunColumn) && missing.Contains(ProductMzColumn)) + { + throw new InvalidDataException( + $"'{Path.GetFileName(path)}' looks like a DIA-NN report, not a spectral library. The library " + + "is normally named report-lib.parquet and carries Product.Mz and Fragment.Type."); + } + + throw new InvalidDataException( + $"Missing required columns in {Path.GetFileName(path)}: {string.Join(", ", missing)}"); + } + + private static DataField Field(DataField[] fields, string name) => + fields.FirstOrDefault(f => string.Equals(f.Name, name, StringComparison.Ordinal)) + ?? throw new InvalidDataException($"Column '{name}' is missing."); + + private static string[] ReadStrings(ParquetRowGroupReader reader, DataField[] fields, string name) + { + DataColumn column = reader.ReadColumnAsync(Field(fields, name)).GetAwaiter().GetResult(); + Array data = column.Data; + var result = new string[data.Length]; + for (var i = 0; i < data.Length; i++) result[i] = data.GetValue(i)?.ToString() ?? string.Empty; + return result; + } + + /// + /// Reads a numeric column regardless of the physical type DIA-NN chose for it. Column + /// types drift between DIA-NN versions, so binding to one would break on upgrade. + /// + private static double[] ReadDoubles(ParquetRowGroupReader reader, DataField[] fields, string name) + { + DataColumn column = reader.ReadColumnAsync(Field(fields, name)).GetAwaiter().GetResult(); + Array data = column.Data; + var result = new double[data.Length]; + + for (var i = 0; i < data.Length; i++) + { + object? value = data.GetValue(i); + result[i] = value switch + { + null => double.NaN, + double d => d, + float f => f, + int n => n, + long l => l, + short s => s, + byte b => b, + decimal m => (double)m, + string text => double.TryParse(text, System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out double parsed) + ? parsed + : double.NaN, + _ => double.NaN, + }; + } + + return result; + } +} diff --git a/dotnet/MARS.IO/MARS.IO.csproj b/dotnet/MARS.IO/MARS.IO.csproj new file mode 100644 index 0000000..f781092 --- /dev/null +++ b/dotnet/MARS.IO/MARS.IO.csproj @@ -0,0 +1,26 @@ + + + + MARS.IO + MARS.IO + mzML passthrough reader/writer and spectral library readers for MARS. + + + + + + + + + + + + + + + + + diff --git a/dotnet/MARS.IO/MatchDumpWriter.cs b/dotnet/MARS.IO/MatchDumpWriter.cs new file mode 100644 index 0000000..c4c1df5 --- /dev/null +++ b/dotnet/MARS.IO/MatchDumpWriter.cs @@ -0,0 +1,164 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Writes the matched-fragment table as CSV, one row per match. + +using System; +using System.Globalization; +using System.IO; +using System.Text; +using MARS.Core; + +namespace MARS.IO; + +/// +/// Dumps a to CSV. +/// +/// This exists to answer "which peak did MARS actually match, and what did it compute from +/// it" without a debugger. It is also the join point for comparing against another +/// implementation: the scan number and the library fragment index together identify a row +/// uniquely, so two dumps of the same input can be merged and differenced column by column. +/// +/// Values are written round-trip ("R") so a re-read loses nothing. Rows come out in match +/// order, which is deterministic for a given input. +/// +public static class MatchDumpWriter +{ + /// Columns that identify the row, written before the feature columns. + private static readonly string[] KeyColumns = + { + "scan_number", "retention_time", "entry_index", "fragment_index", + "peptide_group", "peptide", "ion_annotation", "expected_mz", "observed_mz", + "delta_mz", "observed_intensity", + }; + + /// + /// Optional per-row model predictions, parallel to the table's rows. When supplied, + /// two more columns are written: the predicted correction and the residual left after + /// applying it. This is what makes the dump comparable against another + /// implementation's model rather than only its features. + /// + /// + /// The table was built without detail columns, so its rows cannot be identified. + /// + public static void Write( + string path, MatchTable table, SpectralLibrary library, double[]? predictions = null) + { + if (!table.KeepDetail) + { + throw new InvalidOperationException( + "The match table was built without detail columns; construct it with " + + "keepDetail: true before dumping."); + } + + // Checked before a file is opened rather than discovered partway through writing + // millions of rows, where the failure is a half-written dump and an index-out-of-range + // with nothing in it naming the cause. + if (predictions is not null && predictions.Length != table.Count) + { + throw new ArgumentException( + $"The match table has {table.Count:N0} rows but {predictions.Length:N0} " + + "predictions were supplied. They have to be parallel: each row's prediction is " + + "written beside it and its residual computed from it.", + nameof(predictions)); + } + + string? directory = Path.GetDirectoryName(Path.GetFullPath(path)); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + + // No BOM: this file exists to be read by other tools, and a BOM makes the first + // column name compare unequal to "scan_number" in most CSV readers. + using var writer = new StreamWriter(path, append: false, new UTF8Encoding(false)); + + // A large cohort produces millions of rows, so this streams and reuses one + // builder rather than composing strings per row. + var line = new StringBuilder(512); + + for (int i = 0; i < KeyColumns.Length; i++) + { + if (i > 0) line.Append(','); + line.Append(KeyColumns[i]); + } + + foreach (MarsFeature feature in table.Collected) + { + line.Append(','); + line.Append(MarsFeatures.NameOf(feature)); + } + + if (predictions is not null) line.Append(",predicted_delta_mz,residual"); + + writer.WriteLine(line.ToString()); + + int[] scanNumber = table.ScanNumber!.Items; + int[] entryIndex = table.LibraryEntryIndex!.Items; + int[] fragmentIndex = table.FragmentIndex!.Items; + double[] observedMz = table.ObservedMz!.Items; + double[] retentionTime = table.RetentionTime!.Items; + double[] deltaMz = table.DeltaMz.Items; + double[] observedIntensity = table.ObservedIntensity.Items; + + for (int row = 0; row < table.Count; row++) + { + line.Clear(); + int entry = entryIndex[row]; + int fragment = fragmentIndex[row]; + + line.Append(scanNumber[row].ToString(CultureInfo.InvariantCulture)); + Append(line, retentionTime[row]); + line.Append(','); + line.Append(entry.ToString(CultureInfo.InvariantCulture)); + line.Append(','); + line.Append(fragment.ToString(CultureInfo.InvariantCulture)); + line.Append(','); + // The peptide, as the dense id folds are dealt over. Emitted so another + // implementation can reproduce exactly the same split rather than approximate it. + line.Append(table.PeptideGroup.Items[row].ToString(CultureInfo.InvariantCulture)); + line.Append(','); + AppendQuoted(line, library.ModifiedSequence is null ? string.Empty : library.ModifiedSequence[entry]); + line.Append(','); + AppendAnnotation(line, library, fragment); + Append(line, library.FragmentMz[fragment]); + Append(line, observedMz[row]); + Append(line, deltaMz[row]); + Append(line, observedIntensity[row]); + + foreach (MarsFeature feature in table.Collected) + Append(line, table.Column(feature).Items[row]); + + if (predictions is not null) + { + Append(line, predictions[row]); + Append(line, deltaMz[row] - predictions[row]); + } + + writer.WriteLine(line.ToString()); + } + } + + private static void Append(StringBuilder line, double value) + { + line.Append(','); + // NaN is meaningful here: it is how an undefined ratio reaches the model, and the + // row-selection step drops on it. Write it rather than blanking it. + line.Append(value.ToString("R", CultureInfo.InvariantCulture)); + } + + /// Reproduces the library's annotation form, e.g. y7+1. + private static void AppendAnnotation(StringBuilder line, SpectralLibrary library, int fragment) + { + line.Append((char)library.FragmentIonType[fragment]); + line.Append(library.FragmentIonNumber[fragment].ToString(CultureInfo.InvariantCulture)); + line.Append('+'); + line.Append(library.FragmentCharge[fragment].ToString(CultureInfo.InvariantCulture)); + } + + /// + /// Peptide sequences carry modification syntax that can contain a comma, so this field + /// is always quoted. + /// + private static void AppendQuoted(StringBuilder line, string value) + { + line.Append('"'); + line.Append(value.Replace("\"", "\"\"", StringComparison.Ordinal)); + line.Append('"'); + } +} diff --git a/dotnet/MARS.IO/MzMLBinaryCodec.cs b/dotnet/MARS.IO/MzMLBinaryCodec.cs new file mode 100644 index 0000000..b2f04f4 --- /dev/null +++ b/dotnet/MARS.IO/MzMLBinaryCodec.cs @@ -0,0 +1,193 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// base64 + zlib codec for mzML binary data arrays. + +using System; +using System.Buffers.Text; +using System.IO; +using System.IO.Compression; +using System.Runtime.InteropServices; +using System.Text; + +namespace MARS.IO; + +/// How one binary data array is stored in the file. +public readonly struct BinaryArrayEncoding +{ + public BinaryArrayEncoding(bool is64Bit, bool zlib) + { + Is64Bit = is64Bit; + Zlib = zlib; + } + + /// True for MS:1000523 (64-bit float), false for MS:1000521 (32-bit float). + public bool Is64Bit { get; } + + /// True when MS:1000574 (zlib compression) is present. + public bool Zlib { get; } + + public override string ToString() => (Is64Bit ? "64-bit" : "32-bit") + (Zlib ? " zlib" : " uncompressed"); +} + +/// +/// Decodes and re-encodes mzML binary data arrays. +/// +/// Encoding is read per ARRAY, never per spectrum: m/z is typically 64-bit while intensity +/// is often 32-bit, and compression can differ between two arrays of the same spectrum. +/// A modified array is always re-encoded with exactly the precision and compression it was +/// decoded with. +/// +/// +public static class MzMLBinaryCodec +{ + /// + /// Decodes a base64 payload into doubles, growing the buffers as needed. Returns the + /// number of values written. + /// + /// Holds the base64-decoded bytes. + /// + /// Holds the inflated bytes. This MUST be a different array from + /// : the compressed bytes are read out of a + /// MemoryStream that wraps the source array without copying it, so inflating in place + /// overwrites compressed data that has not been consumed yet. The corruption only + /// appears once a payload exceeds the decompressor's internal buffer, which is why it + /// hides on small spectra and surfaces on large ones. + /// + public static int Decode( + ReadOnlySpan base64Utf8, + BinaryArrayEncoding encoding, + ref byte[] base64Scratch, + ref byte[] inflateScratch, + ref double[] values) + { + int rawLength = DecodeBase64(base64Utf8, ref base64Scratch); + byte[] raw = base64Scratch; + + if (encoding.Zlib) + { + rawLength = Inflate(base64Scratch, rawLength, ref inflateScratch); + raw = inflateScratch; + } + + int bytesPerValue = encoding.Is64Bit ? 8 : 4; + if (rawLength % bytesPerValue != 0) + { + throw new InvalidDataException( + $"Binary array length {rawLength} is not a multiple of {bytesPerValue} for a {encoding} array."); + } + + int count = rawLength / bytesPerValue; + if (values.Length < count) values = new double[Math.Max(count, 1024)]; + + // mzML binary arrays are little-endian by specification. + if (encoding.Is64Bit) + { + ReadOnlySpan source = MemoryMarshal.Cast(raw.AsSpan(0, rawLength)); + source.CopyTo(values.AsSpan(0, count)); + } + else + { + ReadOnlySpan source = MemoryMarshal.Cast(raw.AsSpan(0, rawLength)); + for (int i = 0; i < count; i++) values[i] = source[i]; + } + + return count; + } + + /// + /// Encodes values back to base64 with the same precision and compression, writing UTF-8 + /// base64 into . Returns the base64 CHARACTER count, which + /// is what the encodedLength attribute records. + /// + public static int Encode( + ReadOnlySpan values, + BinaryArrayEncoding encoding, + ref byte[] rawScratch, + ref byte[] deflateScratch, + ref byte[] base64Utf8) + { + int bytesPerValue = encoding.Is64Bit ? 8 : 4; + int rawLength = values.Length * bytesPerValue; + if (rawScratch.Length < rawLength) rawScratch = new byte[Math.Max(rawLength, 4096)]; + + if (encoding.Is64Bit) + { + values.CopyTo(MemoryMarshal.Cast(rawScratch.AsSpan(0, rawLength))); + } + else + { + Span destination = MemoryMarshal.Cast(rawScratch.AsSpan(0, rawLength)); + for (int i = 0; i < values.Length; i++) destination[i] = (float)values[i]; + } + + byte[] payload = rawScratch; + int payloadLength = rawLength; + + if (encoding.Zlib) + { + payloadLength = Deflate(rawScratch, rawLength, ref deflateScratch); + payload = deflateScratch; + } + + int base64Length = Base64.GetMaxEncodedToUtf8Length(payloadLength); + if (base64Utf8.Length < base64Length) base64Utf8 = new byte[Math.Max(base64Length, 4096)]; + + System.Buffers.OperationStatus status = Base64.EncodeToUtf8( + payload.AsSpan(0, payloadLength), base64Utf8, out _, out int written, isFinalBlock: true); + if (status != System.Buffers.OperationStatus.Done) + throw new InvalidDataException($"base64 encoding failed with status {status}."); + + return written; + } + + private static int DecodeBase64(ReadOnlySpan base64Utf8, ref byte[] destination) + { + int maxLength = (base64Utf8.Length / 4 * 3) + 3; + if (destination.Length < maxLength) destination = new byte[Math.Max(maxLength, 4096)]; + + System.Buffers.OperationStatus status = Base64.DecodeFromUtf8( + base64Utf8, destination, out int consumed, out int written, isFinalBlock: true); + if (status == System.Buffers.OperationStatus.Done && consumed == base64Utf8.Length) return written; + + // The fast path rejects embedded whitespace, which the mzML specification permits + // inside element content even though pwiz does not emit it. Fall back to the + // whitespace-tolerant decoder. + string text = Encoding.UTF8.GetString(base64Utf8); + byte[] decoded = Convert.FromBase64String(text); + if (destination.Length < decoded.Length) destination = new byte[decoded.Length]; + decoded.CopyTo(destination, 0); + return decoded.Length; + } + + private static int Inflate(byte[] source, int length, ref byte[] destination) + { + // mzML uses the zlib container, with its 2-byte header and Adler-32 trailer, not a + // raw deflate stream. ZLibStream, never DeflateStream. + using var input = new MemoryStream(source, 0, length, writable: false); + using var zlib = new ZLibStream(input, CompressionMode.Decompress); + + int total = 0; + while (true) + { + if (total == destination.Length) Array.Resize(ref destination, Math.Max(destination.Length * 2, 8192)); + int read = zlib.Read(destination, total, destination.Length - total); + if (read == 0) break; + total += read; + } + + return total; + } + + private static int Deflate(byte[] source, int length, ref byte[] destination) + { + using var output = new MemoryStream(Math.Max(length / 2, 256)); + using (var zlib = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + zlib.Write(source, 0, length); + } + + int compressed = (int)output.Length; + if (destination.Length < compressed) destination = new byte[Math.Max(compressed, 4096)]; + output.GetBuffer().AsSpan(0, compressed).CopyTo(destination); + return compressed; + } +} diff --git a/dotnet/MARS.IO/MzMLComparer.cs b/dotnet/MARS.IO/MzMLComparer.cs new file mode 100644 index 0000000..f022f7d --- /dev/null +++ b/dotnet/MARS.IO/MzMLComparer.cs @@ -0,0 +1,148 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Compares two mzML files on DECODED array values. +// +// Equivalence is deliberately not defined on file bytes: .NET's zlib and Python's zlib +// produce different compressed output at the same nominal level, and any writer may differ +// in whitespace. What has to match is what a consumer actually reads back. + +using System; +using System.Collections.Generic; +using System.IO; +using MARS.Core; + +namespace MARS.IO; + +public sealed class MzMLComparison +{ + public long SpectraCompared { get; set; } + + public long SpectraOnlyInA { get; set; } + + public long SpectraOnlyInB { get; set; } + + /// + /// True when the two files stopped holding the same spectra in the same order, so the + /// comparison ended early and the counts describe only what preceded it. + /// + public bool Diverged { get; set; } + + public long MzValuesCompared { get; set; } + + public long MzValuesDiffering { get; set; } + + public long IntensityValuesDiffering { get; set; } + + public double MaxAbsoluteMzDifference { get; set; } + + public double MaxAbsoluteIntensityDifference { get; set; } + + public List Problems { get; } = new(); + + public bool MzBitIdentical => MzValuesDiffering == 0 && SpectraOnlyInA == 0 && SpectraOnlyInB == 0; + + public bool IntensityBitIdentical => IntensityValuesDiffering == 0; +} + +public static class MzMLComparer +{ + /// + /// Streams both files in parallel, pairing spectra by position and checking that the ids + /// agree, then compares decoded m/z and intensity arrays bit for bit. + /// + /// + /// Positional, not a merge by id: this compares a file against a correction of itself, or + /// two corrections of one input, where the spectra are the same set in the same order. It + /// does not realign. One inserted or removed spectrum therefore puts everything after it + /// out of step, and the run says so and stops rather than reporting a tally that counts + /// every remaining pair as a difference - which would read as "these files are unrelated" + /// when the truth is "these files differ by one spectrum". + /// + /// Cap on the detail list; counters stay exact. + public static MzMLComparison Compare(string pathA, string pathB, int maxProblemsReported = 20) + { + var result = new MzMLComparison(); + + MzMLFileInfo infoA = MzMLFile.Inspect(pathA); + MzMLFileInfo infoB = MzMLFile.Inspect(pathB); + + using IEnumerator a = MzMLFile.ReadSpectra(infoA, msLevel: null).GetEnumerator(); + using IEnumerator b = MzMLFile.ReadSpectra(infoB, msLevel: null).GetEnumerator(); + + while (true) + { + bool hasA = a.MoveNext(); + bool hasB = b.MoveNext(); + + if (!hasA && !hasB) break; + if (!hasA) + { + result.SpectraOnlyInB++; + continue; + } + + if (!hasB) + { + result.SpectraOnlyInA++; + continue; + } + + SpectrumRecord left = a.Current; + SpectrumRecord right = b.Current; + + if (!string.Equals(left.Id, right.Id, StringComparison.Ordinal)) + { + // Everything after this point is compared against the wrong spectrum, so the + // counts would stop meaning anything. Report where alignment was lost and stop. + result.Problems.Add( + $"spectrum id mismatch at position {result.SpectraCompared:N0}: '{left.Id}' " + + $"vs '{right.Id}'. The files do not hold the same spectra in the same order, " + + "so the comparison stops here; counts cover the spectra up to this point."); + result.Diverged = true; + result.SpectraOnlyInA++; + result.SpectraOnlyInB++; + break; + } + + result.SpectraCompared++; + + if (left.PeakCount != right.PeakCount) + { + if (result.Problems.Count < maxProblemsReported) + result.Problems.Add($"{left.Id}: peak count {left.PeakCount} vs {right.PeakCount}"); + continue; + } + + ReadOnlySpan mzA = left.Mz; + ReadOnlySpan mzB = right.Mz; + ReadOnlySpan intensityA = left.Intensity; + ReadOnlySpan intensityB = right.Intensity; + + for (int i = 0; i < left.PeakCount; i++) + { + result.MzValuesCompared++; + + if (BitConverter.DoubleToInt64Bits(mzA[i]) != BitConverter.DoubleToInt64Bits(mzB[i])) + { + result.MzValuesDiffering++; + double difference = Math.Abs(mzA[i] - mzB[i]); + if (difference > result.MaxAbsoluteMzDifference) result.MaxAbsoluteMzDifference = difference; + if (result.Problems.Count < maxProblemsReported) + { + result.Problems.Add( + $"{left.Id} peak {i}: m/z {mzA[i]:R} vs {mzB[i]:R}"); + } + } + + if (BitConverter.DoubleToInt64Bits(intensityA[i]) != BitConverter.DoubleToInt64Bits(intensityB[i])) + { + result.IntensityValuesDiffering++; + double difference = Math.Abs(intensityA[i] - intensityB[i]); + if (difference > result.MaxAbsoluteIntensityDifference) + result.MaxAbsoluteIntensityDifference = difference; + } + } + } + + return result; + } +} diff --git a/dotnet/MARS.IO/MzMLFile.cs b/dotnet/MARS.IO/MzMLFile.cs new file mode 100644 index 0000000..e1bf450 --- /dev/null +++ b/dotnet/MARS.IO/MzMLFile.cs @@ -0,0 +1,383 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Passthrough mzML reader and writer. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using MARS.Core; + +namespace MARS.IO; + +/// What MARS needs to know about a file before streaming it. +public sealed class MzMLFileInfo +{ + public required string Path { get; init; } + + public required long Length { get; init; } + + /// Run startTimeStamp as a Unix timestamp in seconds, or null when absent. + public required double? AcquisitionStartTime { get; init; } + + /// + /// Byte offset where the copied content stops: the start of the existing indexList, or + /// the end of the mzML element for an unindexed file. Everything before it is preserved + /// byte for byte; everything after it is regenerated. + /// + public required long ContentCutOffset { get; init; } + + public required bool WasIndexed { get; init; } + + /// + /// True when the root element is indexedmzML. A plain mzML has nowhere to put an index, + /// so MARS copies it through unindexed rather than producing invalid XML. + /// + public required bool IsIndexedMzML { get; init; } + + /// + /// Measuring analyzer accession per instrumentConfiguration id, from the file header. + /// Empty when the header did not say. + /// + public IReadOnlyDictionary AnalyzerByConfiguration { get; init; } = + new Dictionary(); + + /// + /// The run's defaultInstrumentConfigurationRef, used by spectra that do not name one. + /// + public string? DefaultConfiguration { get; init; } +} + +public static class MzMLFile +{ + private const int HeaderProbeBytes = 512 * 1024; + private const int TailProbeBytes = 64 * 1024; + + /// + /// Reads the run header and locates where the regenerated trailer begins, without + /// scanning the body. + /// + public static MzMLFileInfo Inspect(string path) + { + var file = new FileInfo(path); + if (!file.Exists) throw new FileNotFoundException("mzML file not found.", path); + + double? acquisitionStart = null; + string? defaultConfiguration = null; + Dictionary analyzers; + bool indexedRoot; + using (FileStream stream = File.OpenRead(path)) + { + int probeLength = (int)Math.Min(HeaderProbeBytes, file.Length); + var probe = new byte[probeLength]; + int read = stream.Read(probe, 0, probeLength); + string header = Encoding.UTF8.GetString(probe, 0, read); + + indexedRoot = header.Contains("= 0) + { + int start = refAt + configMarker.Length; + int end = header.IndexOf('"', start); + if (end > start) defaultConfiguration = header[start..end]; + } + + const string marker = "startTimeStamp=\""; + int at = header.IndexOf(marker, StringComparison.Ordinal); + if (at >= 0) + { + int start = at + marker.Length; + int end = header.IndexOf('"', start); + if (end > start) acquisitionStart = MzMLSpectrumParser.ParseStartTimeStamp(header[start..end]); + } + } + + (long cut, bool indexed) = FindContentCut(path, file.Length); + + return new MzMLFileInfo + { + Path = path, + Length = file.Length, + AcquisitionStartTime = acquisitionStart, + ContentCutOffset = cut, + WasIndexed = indexed, + IsIndexedMzML = indexedRoot, + AnalyzerByConfiguration = analyzers, + DefaultConfiguration = defaultConfiguration, + }; + } + + /// + /// Reads instrumentConfiguration ids and the analyzer each one measures with, out of the + /// header text already in hand. + /// + /// + /// Deliberately a scan of the header probe rather than an XML parse: this runs on files + /// of several gigabytes whose header MARS otherwise only reads to find two attributes, + /// and a configuration list that falls outside the probe is a missing answer rather than + /// a wrong one - detection degrades to "unknown" and the caller keeps its default. + /// + private static Dictionary ParseAnalyzers(string header) + { + var byConfiguration = new Dictionary(StringComparer.Ordinal); + + const string open = "= 0) + { + int idStart = at + open.Length; + int idEnd = header.IndexOf('"', idStart); + if (idEnd < 0) break; + + string id = header[idStart..idEnd]; + int close = header.IndexOf("", idEnd, StringComparison.Ordinal); + int next = header.IndexOf(open, idEnd, StringComparison.Ordinal); + if (close < 0) close = next >= 0 ? next : header.Length; + + var analyzers = new List<(int Order, string Accession)>(); + foreach ((int order, string accession) in ScanAnalyzers(header, idEnd, close)) + analyzers.Add((order, accession)); + + if (MassAnalyzers.MeasuringAnalyzer(analyzers) is string measuring) + byConfiguration[id] = measuring; + + at = next; + } + + return byConfiguration; + } + + private static IEnumerable<(int Order, string Accession)> ScanAnalyzers(string header, int from, int to) + { + const string open = "= 0 && at < to) + { + int orderStart = at + open.Length; + int orderEnd = header.IndexOf('"', orderStart); + if (orderEnd < 0 || orderEnd > to) yield break; + + int close = header.IndexOf("", orderEnd, StringComparison.Ordinal); + if (close < 0 || close > to) close = to; + + if (int.TryParse(header.AsSpan(orderStart, orderEnd - orderStart), NumberStyles.Integer, + CultureInfo.InvariantCulture, out int order)) + { + // The first accession inside the element is the analyzer type; anything after + // it describes the analyzer rather than naming it. + const string accessionMarker = "accession=\""; + int accessionAt = header.IndexOf(accessionMarker, orderEnd, StringComparison.Ordinal); + if (accessionAt >= 0 && accessionAt < close) + { + int accessionStart = accessionAt + accessionMarker.Length; + int accessionEnd = header.IndexOf('"', accessionStart); + if (accessionEnd > accessionStart) + yield return (order, header[accessionStart..accessionEnd]); + } + } + + at = header.IndexOf(open, close, StringComparison.Ordinal); + } + } + + /// + /// Works out which analyzer recorded this run's MS2 spectra - the ones MARS calibrates. + /// + /// + /// The run's default configuration is not the answer on its own. A file from an Orbitrap + /// Astral names the orbitrap as the run default because that is what takes the MS1 + /// survey, and points each MS2 spectrum at a second configuration for the Astral + /// analyzer. Reading only the header would classify such a run by its MS1 analyzer, which + /// on a hybrid instrument is exactly the wrong one. + /// + public static MassAnalyzerClass DetectMs2Analyzer(MzMLFileInfo info) + { + if (info.AnalyzerByConfiguration.Count == 0) return MassAnalyzerClass.Unknown; + + // One configuration means every spectrum used it, and no spectrum needs reading. + if (info.AnalyzerByConfiguration.Count == 1) + { + foreach (string accession in info.AnalyzerByConfiguration.Values) + return MassAnalyzers.Classify(accession); + } + + foreach (SpectrumRecord record in ReadSpectra(info, msLevel: 2)) + { + string? configuration = record.InstrumentConfigurationRef ?? info.DefaultConfiguration; + if (configuration is not null && + info.AnalyzerByConfiguration.TryGetValue(configuration, out string? accession)) + { + return MassAnalyzers.Classify(accession); + } + + // The file has several configurations but this spectrum does not say which, so + // the header cannot settle it. Thermo's filter string can. + return MassAnalyzers.ClassifyFilterString(record.FilterString); + } + + return MassAnalyzerClass.Unknown; + } + + /// + /// Finds where the preserved content ends. An indexed file records the offset of its own + /// indexList in the trailer, so this costs one small read rather than a scan. + /// + private static (long Cut, bool Indexed) FindContentCut(string path, long fileLength) + { + int tailLength = (int)Math.Min(TailProbeBytes, fileLength); + var tail = new byte[tailLength]; + using (FileStream stream = File.OpenRead(path)) + { + stream.Seek(fileLength - tailLength, SeekOrigin.Begin); + ReadExactly(stream, tail, tailLength); + } + + string text = Encoding.UTF8.GetString(tail); + const string offsetOpen = ""; + const string offsetClose = ""; + int at = text.LastIndexOf(offsetOpen, StringComparison.Ordinal); + if (at >= 0) + { + int start = at + offsetOpen.Length; + int end = text.IndexOf(offsetClose, start, StringComparison.Ordinal); + if (end > start && + long.TryParse(text.AsSpan(start, end - start).Trim(), NumberStyles.Integer, + CultureInfo.InvariantCulture, out long recorded) && + recorded > 0 && recorded < fileLength) + { + long cutAt = FindIndexListAt(path, recorded); + if (cutAt >= 0) return (cutAt, true); + } + } + + // Unindexed, or the recorded offset does not point at an indexList. Fall back to the + // end of the mzML element and write an index the file did not previously have. + const string mzmlClose = ""; + int closeAt = text.LastIndexOf(mzmlClose, StringComparison.Ordinal); + if (closeAt < 0) + throw new InvalidDataException($"No found near the end of {path}; this does not look like an mzML file."); + + long cut = fileLength - tailLength + Encoding.UTF8.GetByteCount(text[..(closeAt + mzmlClose.Length)]); + return (cut, false); + } + + /// + /// Confirms the recorded offset really points at an indexList and returns the offset of + /// its opening angle bracket. Some writers record the start of the indentation rather + /// than the tag, so leading whitespace is skipped and kept in the preserved content. + /// Returns -1 when there is no indexList there. + /// + private static long FindIndexListAt(string path, long offset) + { + using FileStream stream = File.OpenRead(path); + stream.Seek(offset, SeekOrigin.Begin); + var probe = new byte[32]; + int read = stream.Read(probe, 0, probe.Length); + if (read <= 0) return -1; + + int skipped = 0; + while (skipped < read && + (probe[skipped] == (byte)' ' || probe[skipped] == (byte)'\t' || + probe[skipped] == (byte)'\r' || probe[skipped] == (byte)'\n')) + { + skipped++; + } + + string text = Encoding.UTF8.GetString(probe, skipped, read - skipped); + return text.StartsWith(" + /// Streams spectra. The arrays on the yielded record are reused between iterations, so + /// a consumer that needs to keep them must copy. + /// + public static IEnumerable ReadSpectra(MzMLFileInfo info, int? msLevel = 2) + { + using FileStream input = new(info.Path, FileMode.Open, FileAccess.Read, FileShare.Read, + 1 << 20, FileOptions.SequentialScan); + using var scanner = new MzMLSpanScanner(input, info.ContentCutOffset); + + var context = new DecodeContext(); + + while (scanner.TryReadRegion(out MzMLRegionKind kind, out int start, out int length)) + { + if (kind != MzMLRegionKind.Spectrum) + { + scanner.Advance(length); + continue; + } + + ParsedSpectrum parsed = MzMLSpectrumParser.Parse(scanner.Buffer, start, length); + SpectrumRecord record = parsed.Record; + + bool wanted = !msLevel.HasValue || record.MsLevel == msLevel.Value; + if (wanted && context.Decode(scanner.Buffer, start, parsed, record)) + { + record.AcquisitionStartTime = info.AcquisitionStartTime; + record.AbsoluteTime = info.AcquisitionStartTime is double t + ? t + (record.RetentionTime * 60.0) + : record.RetentionTime * 60.0; + + yield return record; + } + + scanner.Advance(length); + } + } + + internal static void ReadExactly(Stream stream, byte[] buffer, int count) + { + int total = 0; + while (total < count) + { + int read = stream.Read(buffer, total, count - total); + if (read <= 0) throw new EndOfStreamException(); + total += read; + } + } + + /// Scratch buffers for decoding one spectrum's arrays. + internal sealed class DecodeContext + { + private byte[] _base64 = new byte[1 << 16]; + private byte[] _inflated = new byte[1 << 16]; + private double[] _mz = new double[4096]; + private double[] _intensity = new double[4096]; + + /// + /// Decodes the m/z and intensity arrays into the record. Returns false when the + /// spectrum carries no peaks. + /// + public bool Decode(byte[] buffer, int spanStart, ParsedSpectrum parsed, SpectrumRecord record) + { + if (parsed.MzArrayIndex < 0 || parsed.IntensityArrayIndex < 0) return false; + + BinaryArrayLocation mzArray = parsed.Arrays[parsed.MzArrayIndex]; + BinaryArrayLocation intensityArray = parsed.Arrays[parsed.IntensityArrayIndex]; + if (mzArray.BinaryTextStart < 0 || intensityArray.BinaryTextStart < 0) return false; + + int mzCount = MzMLBinaryCodec.Decode( + new ReadOnlySpan(buffer, spanStart + mzArray.BinaryTextStart, mzArray.BinaryTextLength), + mzArray.Encoding, ref _base64, ref _inflated, ref _mz); + + int intensityCount = MzMLBinaryCodec.Decode( + new ReadOnlySpan(buffer, spanStart + intensityArray.BinaryTextStart, intensityArray.BinaryTextLength), + intensityArray.Encoding, ref _base64, ref _inflated, ref _intensity); + + if (mzCount == 0 || mzCount != intensityCount) return false; + + record.MzArray = _mz; + record.IntensityArray = _intensity; + record.PeakCount = mzCount; + + double sum = 0; + for (int i = 0; i < mzCount; i++) sum += _intensity[i]; + record.SummedIntensity = sum; + return true; + } + } +} diff --git a/dotnet/MARS.IO/MzMLSpanScanner.cs b/dotnet/MARS.IO/MzMLSpanScanner.cs new file mode 100644 index 0000000..938dd9b --- /dev/null +++ b/dotnet/MARS.IO/MzMLSpanScanner.cs @@ -0,0 +1,276 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Streaming scanner that carves an mzML file into regions without ever holding the file in +// memory: runs of bytes to pass through untouched, and the indexed elements between them. + +using System; +using System.IO; + +namespace MARS.IO; + +internal enum MzMLRegionKind +{ + /// Bytes to copy through untouched. + Gap, + + Spectrum, + + Chromatogram, +} + +/// +/// Walks an mzML file as a sequence of regions. +/// +/// Everything that is not a spectrum or chromatogram comes back as a gap the caller copies +/// verbatim, which is what makes the passthrough contract hold by construction: anything +/// MARS does not deliberately replace is byte-identical to the input, including attribute +/// order, whitespace, namespace declarations and every cvParam. +/// +/// +/// Memory is bounded by the largest single element rather than by the file. A long stretch +/// with no elements is handed back in chunks instead of accumulating. +/// +/// +internal sealed class MzMLSpanScanner : IDisposable +{ + private static readonly byte[] SpectrumOpen = " _buffer; + + /// + /// Produces the next region. The caller must handle it and then call + /// with its length before asking for another. + /// + public bool TryReadRegion(out MzMLRegionKind kind, out int start, out int length) + { + kind = MzMLRegionKind.Gap; + start = _cursor; + length = 0; + + // An element located on a previous call, with gap bytes still in front of it. + if (_pendingElementStart >= 0) + { + if (_cursor < _pendingElementStart) + { + length = _pendingElementStart - _cursor; + return true; + } + + kind = _pendingElementKind; + byte[] closeTag = kind == MzMLRegionKind.Spectrum ? SpectrumClose : ChromatogramClose; + int end = FindElementEnd(closeTag); + if (end < 0) + throw new InvalidDataException($"mzML ended in the middle of a {kind} element."); + + _pendingElementStart = -1; + start = _cursor; + length = end - _cursor; + return true; + } + + int elementStart = FindElementStart(out MzMLRegionKind found, out int gapChunk); + if (elementStart < 0) + { + if (gapChunk > 0) + { + start = _cursor; + length = gapChunk; + return true; + } + + return false; + } + + if (elementStart > _cursor) + { + _pendingElementStart = elementStart; + _pendingElementKind = found; + start = _cursor; + length = elementStart - _cursor; + return true; + } + + kind = found; + byte[] close = found == MzMLRegionKind.Spectrum ? SpectrumClose : ChromatogramClose; + int elementEnd = FindElementEnd(close); + if (elementEnd < 0) + throw new InvalidDataException($"mzML ended in the middle of a {found} element."); + + start = _cursor; + length = elementEnd - _cursor; + return true; + } + + /// Consumes bytes the caller has finished with. + public void Advance(int count) + { + _cursor += count; + _consumedFromFile += count; + } + + public void Dispose() + { + } + + // Search positions are tracked RELATIVE to _cursor, because FillMore compacts the + // buffer down to _cursor == 0 and an absolute index would silently go stale. + private int FindElementStart(out MzMLRegionKind kind, out int gapChunk) + { + kind = MzMLRegionKind.Gap; + gapChunk = 0; + int relative = 0; + + while (true) + { + int best = -1; + MzMLRegionKind bestKind = MzMLRegionKind.Gap; + bool needMore = false; + + foreach ((byte[] tag, MzMLRegionKind tagKind) in new[] + { + (SpectrumOpen, MzMLRegionKind.Spectrum), + (ChromatogramOpen, MzMLRegionKind.Chromatogram), + }) + { + int searchFrom = _cursor + relative; + while (true) + { + int hit = IndexOf(searchFrom, tag); + if (hit < 0) break; + + int after = hit + tag.Length; + if (after >= _dataEnd) + { + // Cannot yet tell ' || next == (byte)'/') + { + if (best < 0 || hit < best) + { + best = hit; + bestKind = tagKind; + } + + break; + } + + searchFrom = after; + } + } + + if (best >= 0) + { + kind = bestKind; + return best; + } + + int resident = _dataEnd - _cursor; + if (!needMore && resident > MaxGapChunk) + { + // Hand back part of the gap so a long run without elements does not grow + // the resident window without bound. + gapChunk = resident - MaxOpenTagLength; + return -1; + } + + // Everything except the last few bytes has now been searched and rejected. Work + // this out BEFORE reading more, or the freshly read window gets skipped whole. + int searchedTo = Math.Max(0, resident - MaxOpenTagLength); + + if (!FillMore()) + { + gapChunk = _dataEnd - _cursor; + return -1; + } + + relative = searchedTo; + } + } + + private int FindElementEnd(byte[] closeTag) + { + int relative = 0; + while (true) + { + int found = IndexOf(_cursor + relative, closeTag); + if (found >= 0) return found + closeTag.Length; + + relative = Math.Max(0, (_dataEnd - _cursor) - (closeTag.Length - 1)); + if (!FillMore()) return -1; + } + } + + private int IndexOf(int from, byte[] needle) + { + if (from >= _dataEnd) return -1; + int found = _buffer.AsSpan(from, _dataEnd - from).IndexOf(needle); + return found < 0 ? -1 : found + from; + } + + /// + /// Reads more input, compacting first and growing the buffer only when the resident + /// window genuinely needs to be larger (one very large element). + /// + private bool FillMore() + { + if (_inputExhausted) return false; + + if (_cursor > 0) + { + int live = _dataEnd - _cursor; + if (live > 0) Array.Copy(_buffer, _cursor, _buffer, 0, live); + _dataEnd = live; + if (_pendingElementStart >= 0) _pendingElementStart -= _cursor; + _cursor = 0; + } + + if (_dataEnd == _buffer.Length) Array.Resize(ref _buffer, _buffer.Length * 2); + + long remainingInFile = _limit - (_consumedFromFile + _dataEnd); + if (remainingInFile <= 0) + { + _inputExhausted = true; + return false; + } + + int want = (int)Math.Min(_buffer.Length - _dataEnd, remainingInFile); + int read = _input.Read(_buffer, _dataEnd, want); + if (read <= 0) + { + _inputExhausted = true; + return false; + } + + _dataEnd += read; + return true; + } +} diff --git a/dotnet/MARS.IO/MzMLSpectrumParser.cs b/dotnet/MARS.IO/MzMLSpectrumParser.cs new file mode 100644 index 0000000..c654b14 --- /dev/null +++ b/dotnet/MARS.IO/MzMLSpectrumParser.cs @@ -0,0 +1,388 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Parses one element out of an mzML byte span. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Xml; +using MARS.Core; + +namespace MARS.IO; + +/// Where one binaryDataArray lives inside a spectrum span, in bytes relative to it. +public sealed class BinaryArrayLocation +{ + public BinaryArrayEncoding Encoding; + + public bool IsMzArray; + + public bool IsIntensityArray; + + /// Offset of the encodedLength attribute's value, or -1 when absent. + public int EncodedLengthStart = -1; + + public int EncodedLengthLength; + + /// Offset of the text inside the binary element, or -1 for an empty element. + public int BinaryTextStart = -1; + + public int BinaryTextLength; +} + +/// Everything MARS needs from one spectrum, plus the byte ranges to splice. +public sealed class ParsedSpectrum +{ + public readonly SpectrumRecord Record = new(); + + public readonly List Arrays = new(); + + public int MzArrayIndex = -1; + + public int IntensityArrayIndex = -1; + + public int DefaultArrayLength; +} + +/// +/// Reads spectrum metadata by CV ACCESSION rather than by name. Names are display strings +/// that vary between writers; accessions are the contract. It also means the +/// userParam name="ms level" that pwiz writes inside an isolation window cannot be +/// mistaken for the real MS:1000511. +/// +public static class MzMLSpectrumParser +{ + public const string MsLevel = "MS:1000511"; + public const string FilterString = "MS:1000512"; + public const string TotalIonCurrent = "MS:1000285"; + public const string ScanStartTime = "MS:1000016"; + public const string IonInjectionTime = "MS:1000927"; + public const string IsolationWindowTarget = "MS:1000827"; + public const string IsolationWindowLowerOffset = "MS:1000828"; + public const string IsolationWindowUpperOffset = "MS:1000829"; + public const string SelectedIonMz = "MS:1000744"; + public const string Bit32Float = "MS:1000521"; + public const string Bit64Float = "MS:1000523"; + public const string ZlibCompression = "MS:1000574"; + public const string MzArray = "MS:1000514"; + public const string IntensityArray = "MS:1000515"; + public const string UnitMinute = "UO:0000031"; + public const string UnitSecond = "UO:0000010"; + + private static readonly XmlReaderSettings ReaderSettings = new() + { + ConformanceLevel = ConformanceLevel.Fragment, + IgnoreComments = true, + IgnoreProcessingInstructions = true, + IgnoreWhitespace = true, + DtdProcessing = DtdProcessing.Prohibit, + CloseInput = false, + }; + + /// + /// Parses metadata and array encodings from a spectrum span. Binary payloads are NOT + /// decoded here; the caller decides which arrays it needs. + /// + public static ParsedSpectrum Parse(byte[] buffer, int start, int length) + { + var parsed = new ParsedSpectrum(); + SpectrumRecord record = parsed.Record; + + using var stream = new MemoryStream(buffer, start, length, writable: false); + using XmlReader reader = XmlReader.Create(stream, ReaderSettings); + + bool inIsolationWindow = false; + bool haveIsolationWindow = false; + bool inSelectedIon = false; + double target = 0, lowerOffset = 0, upperOffset = 0; + double selectedIonMz = double.NaN; + BinaryArrayLocation? currentArray = null; + + while (reader.Read()) + { + if (reader.NodeType == XmlNodeType.EndElement) + { + switch (reader.LocalName) + { + case "isolationWindow": + inIsolationWindow = false; + break; + case "selectedIon": + inSelectedIon = false; + break; + case "binaryDataArray": + currentArray = null; + break; + } + + continue; + } + + if (reader.NodeType != XmlNodeType.Element) continue; + + switch (reader.LocalName) + { + case "spectrum": + record.Id = reader.GetAttribute("id") ?? string.Empty; + record.Index = ParseInt(reader.GetAttribute("index")); + record.InstrumentConfigurationRef = reader.GetAttribute("instrumentConfigurationRef"); + parsed.DefaultArrayLength = ParseInt(reader.GetAttribute("defaultArrayLength")); + break; + + case "scan": + record.InstrumentConfigurationRef ??= reader.GetAttribute("instrumentConfigurationRef"); + break; + + case "isolationWindow": + inIsolationWindow = true; + break; + + case "selectedIon": + inSelectedIon = true; + break; + + case "binaryDataArray": + currentArray = new BinaryArrayLocation + { + // mzML has no "uncompressed" default marker that must be present, so + // absence of MS:1000574 means no compression. + Encoding = new BinaryArrayEncoding(is64Bit: true, zlib: false), + }; + parsed.Arrays.Add(currentArray); + break; + + case "cvParam": + { + string accession = reader.GetAttribute("accession") ?? string.Empty; + string? value = reader.GetAttribute("value"); + + if (currentArray is not null) + { + switch (accession) + { + case Bit64Float: + currentArray.Encoding = new BinaryArrayEncoding(true, currentArray.Encoding.Zlib); + continue; + case Bit32Float: + currentArray.Encoding = new BinaryArrayEncoding(false, currentArray.Encoding.Zlib); + continue; + case ZlibCompression: + currentArray.Encoding = new BinaryArrayEncoding(currentArray.Encoding.Is64Bit, true); + continue; + case MzArray: + currentArray.IsMzArray = true; + parsed.MzArrayIndex = parsed.Arrays.Count - 1; + continue; + case IntensityArray: + currentArray.IsIntensityArray = true; + parsed.IntensityArrayIndex = parsed.Arrays.Count - 1; + continue; + } + + continue; + } + + switch (accession) + { + case MsLevel: + record.MsLevel = ParseInt(value); + break; + + case FilterString: + record.FilterString = value; + break; + + case TotalIonCurrent: + record.ReportedTic = ParseDouble(value); + break; + + case ScanStartTime: + { + double time = ParseDouble(value); + string unit = reader.GetAttribute("unitAccession") ?? string.Empty; + // Thermo files record minutes; the specification permits seconds. + record.RetentionTime = unit == UnitSecond ? time / 60.0 : time; + break; + } + + case IonInjectionTime: + // The cvParam is milliseconds; MARS works in seconds throughout. + record.InjectionTime = ParseDouble(value) / 1000.0; + break; + + case IsolationWindowTarget when inIsolationWindow: + target = ParseDouble(value); + haveIsolationWindow = true; + break; + + case IsolationWindowLowerOffset when inIsolationWindow: + lowerOffset = ParseDouble(value); + break; + + case IsolationWindowUpperOffset when inIsolationWindow: + upperOffset = ParseDouble(value); + break; + + case SelectedIonMz when inSelectedIon: + if (double.IsNaN(selectedIonMz)) selectedIonMz = ParseDouble(value); + break; + } + + break; + } + } + } + + if (haveIsolationWindow) + { + record.PrecursorMzCenter = target; + record.PrecursorMzLow = target - lowerOffset; + record.PrecursorMzHigh = target + upperOffset; + } + else if (!double.IsNaN(selectedIonMz)) + { + // Fallback used by the Python implementation when no isolation window is written. + record.PrecursorMzCenter = selectedIonMz; + record.PrecursorMzLow = selectedIonMz - 0.5; + record.PrecursorMzHigh = selectedIonMz + 0.5; + } + + record.ScanNumber = ParseScanNumber(record.Id, record.Index); + LocateArrayBytes(buffer, start, length, parsed); + return parsed; + } + + /// + /// Finds the byte ranges of each binaryDataArray's encodedLength value and binary text. + /// + /// This is a byte scan rather than another XmlReader pass because XmlReader reports + /// line and character positions, not byte offsets, and the writer splices bytes. Tag + /// delimiters cannot appear unescaped inside attribute values or base64 content, so + /// scanning for them inside a single spectrum element is unambiguous. + /// + /// + private static void LocateArrayBytes(byte[] buffer, int start, int length, ParsedSpectrum parsed) + { + var span = new ReadOnlySpan(buffer, start, length); + ReadOnlySpan arrayTag = " encodedLengthAttr = "encodedLength=\""u8; + ReadOnlySpan binaryOpen = ""u8; + ReadOnlySpan binaryClose = ""u8; + ReadOnlySpan binaryEmpty = ""u8; + + int cursor = 0; + int arrayIndex = 0; + while (arrayIndex < parsed.Arrays.Count) + { + int tag = IndexOf(span, arrayTag, cursor); + if (tag < 0) break; + + int after = tag + arrayTag.Length; + // Skip , which shares the prefix. + if (after < span.Length && span[after] != (byte)' ' && span[after] != (byte)'>' && + span[after] != (byte)'\r' && span[after] != (byte)'\n' && span[after] != (byte)'\t') + { + cursor = after; + continue; + } + + BinaryArrayLocation location = parsed.Arrays[arrayIndex]; + + int tagEnd = IndexOf(span, ">"u8, after); + if (tagEnd < 0) break; + + int attr = IndexOf(span[..tagEnd], encodedLengthAttr, after); + if (attr >= 0) + { + int valueStart = attr + encodedLengthAttr.Length; + int valueEnd = IndexOf(span, "\""u8, valueStart); + if (valueEnd > valueStart) + { + location.EncodedLengthStart = valueStart; + location.EncodedLengthLength = valueEnd - valueStart; + } + } + + int open = IndexOf(span, binaryOpen, tagEnd); + int empty = IndexOf(span, binaryEmpty, tagEnd); + if (empty >= 0 && (open < 0 || empty < open)) + { + location.BinaryTextStart = -1; + cursor = empty + binaryEmpty.Length; + } + else if (open >= 0) + { + int textStart = open + binaryOpen.Length; + int close = IndexOf(span, binaryClose, textStart); + if (close < 0) break; + location.BinaryTextStart = textStart; + location.BinaryTextLength = close - textStart; + cursor = close + binaryClose.Length; + } + else + { + break; + } + + arrayIndex++; + } + } + + public static int IndexOf(ReadOnlySpan haystack, ReadOnlySpan needle, int from) + { + if (from >= haystack.Length) return -1; + int found = haystack[from..].IndexOf(needle); + return found < 0 ? -1 : found + from; + } + + /// + /// Pulls the scan number out of a Thermo nativeID + /// ("controllerType=0 controllerNumber=1 scan=4321"), falling back to the list index. + /// + public static int ParseScanNumber(string id, int index) + { + const string marker = "scan="; + int at = id.IndexOf(marker, StringComparison.Ordinal); + if (at < 0) return index; + + int start = at + marker.Length; + int end = start; + while (end < id.Length && char.IsDigit(id[end])) end++; + return end > start && int.TryParse(id.AsSpan(start, end - start), NumberStyles.Integer, CultureInfo.InvariantCulture, out int scan) + ? scan + : index; + } + + private static int ParseInt(string? value) => + value is not null && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int result) + ? result + : 0; + + private static double ParseDouble(string? value) => + value is not null && double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double result) + ? result + : 0.0; + + /// Parses the run's startTimeStamp attribute into a Unix timestamp in seconds. + public static double? ParseStartTimeStamp(string timestamp) + { + if (string.IsNullOrWhiteSpace(timestamp)) return null; + string text = timestamp.Trim(); + if (text.EndsWith("Z", StringComparison.Ordinal)) text = text[..^1] + "+00:00"; + + if (DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out DateTimeOffset parsed)) + { + return parsed.ToUnixTimeMilliseconds() / 1000.0; + } + + if (DateTime.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTime plain)) + return new DateTimeOffset(plain.ToUniversalTime(), TimeSpan.Zero).ToUnixTimeMilliseconds() / 1000.0; + + return null; + } + + internal static string GetUtf8String(byte[] buffer, int start, int length) => + Encoding.UTF8.GetString(buffer, start, length); +} diff --git a/dotnet/MARS.IO/MzMLSpectrumSource.cs b/dotnet/MARS.IO/MzMLSpectrumSource.cs new file mode 100644 index 0000000..85e8706 --- /dev/null +++ b/dotnet/MARS.IO/MzMLSpectrumSource.cs @@ -0,0 +1,48 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System.Collections.Generic; +using MARS.Core; + +namespace MARS.IO; + +/// +/// Reads spectra from an mzML, which is what MARS has always done. +/// +/// +/// A thin adapter over . It also carries the +/// , because writing mzML needs it: MARS produces mzML by splicing +/// corrected bytes into a copy of the input, and that needs the byte offsets this inspection +/// found. +/// +public sealed class MzMLSpectrumSource : ISpectrumSource +{ + public MzMLSpectrumSource(MzMLFileInfo info) + { + Info = info; + Analyzer = MzMLFile.DetectMs2Analyzer(info); + } + + public MzMLSpectrumSource(string path) + : this(MzMLFile.Inspect(path)) + { + } + + /// The inspection result, for the byte-splice writer. + public MzMLFileInfo Info { get; } + + public string Path => Info.Path; + + public long Length => Info.Length; + + public double? AcquisitionStartTime => Info.AcquisitionStartTime; + + public MassAnalyzerClass Analyzer { get; } + + public IEnumerable ReadSpectra(int? msLevel = 2) => + MzMLFile.ReadSpectra(Info, msLevel); + + public void Dispose() + { + // Nothing held open: MzMLFile opens and closes a stream per read. + } +} diff --git a/dotnet/MARS.IO/MzMLValidator.cs b/dotnet/MARS.IO/MzMLValidator.cs new file mode 100644 index 0000000..11064e0 --- /dev/null +++ b/dotnet/MARS.IO/MzMLValidator.cs @@ -0,0 +1,263 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Structural checks on a written mzML: the index points where it claims, and the SHA-1 +// covers what the specification says it covers. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; + +namespace MARS.IO; + +public sealed class IndexValidationResult +{ + public bool IsIndexed { get; init; } + + public int SpectrumOffsets { get; init; } + + public int ChromatogramOffsets { get; init; } + + /// Offsets that do not land on the element they claim to. + public List BadOffsets { get; init; } = new(); + + public bool ChecksumPresent { get; init; } + + public bool ChecksumValid { get; init; } + + public string RecordedChecksum { get; init; } = string.Empty; + + public string ComputedChecksum { get; init; } = string.Empty; + + public bool IsValid => BadOffsets.Count == 0 && (!ChecksumPresent || ChecksumValid); +} + +public static class MzMLValidator +{ + /// + /// Verifies that every recorded index offset lands on the start tag of the element it + /// names, and that the SHA-1 fileChecksum matches. + /// + /// 0 checks every offset; a positive value samples evenly. + public static IndexValidationResult Validate(string path, int maxOffsetsChecked = 0) + { + var file = new FileInfo(path); + using FileStream stream = File.OpenRead(path); + + (long indexListOffset, string recordedChecksum, long checksumTagOffset) = ReadTrailer(stream, file.Length); + if (indexListOffset < 0) + { + return new IndexValidationResult + { + IsIndexed = false, + ChecksumPresent = recordedChecksum.Length > 0, + }; + } + + string indexXml = ReadIndexList(stream, indexListOffset, file.Length); + List<(string Name, string Id, long Offset)> entries = ParseIndex(indexXml); + + var spectrumCount = 0; + var chromatogramCount = 0; + foreach ((string name, _, _) in entries) + { + if (name == "spectrum") spectrumCount++; + else if (name == "chromatogram") chromatogramCount++; + } + + var bad = new List(); + var probe = new byte[64]; + int step = maxOffsetsChecked > 0 && entries.Count > maxOffsetsChecked + ? entries.Count / maxOffsetsChecked + : 1; + + for (int i = 0; i < entries.Count; i += step) + { + (string name, string id, long offset) = entries[i]; + if (offset < 0 || offset >= file.Length) + { + bad.Add($"{name} '{id}' offset {offset} is outside the file"); + continue; + } + + stream.Seek(offset, SeekOrigin.Begin); + int read = stream.Read(probe, 0, probe.Length); + string text = Encoding.UTF8.GetString(probe, 0, read); + string expected = "<" + name; + if (!text.StartsWith(expected, StringComparison.Ordinal)) + { + bad.Add($"{name} '{id}' offset {offset} lands on \"{Truncate(text, 24)}\", not {expected}"); + } + } + + bool checksumPresent = recordedChecksum.Length > 0; + string computed = string.Empty; + if (checksumPresent) + { + computed = ComputeChecksum(stream, checksumTagOffset); + } + + return new IndexValidationResult + { + IsIndexed = true, + SpectrumOffsets = spectrumCount, + ChromatogramOffsets = chromatogramCount, + BadOffsets = bad, + ChecksumPresent = checksumPresent, + ChecksumValid = checksumPresent && + string.Equals(computed, recordedChecksum, StringComparison.OrdinalIgnoreCase), + RecordedChecksum = recordedChecksum, + ComputedChecksum = computed, + }; + } + + /// + /// The mzML checksum covers every byte up to AND INCLUDING the fileChecksum opening + /// tag. Verified empirically against pwiz-written files, whose recorded digest only + /// reproduces under that convention. + /// + private static string ComputeChecksum(FileStream stream, long checksumTagOffset) + { + const string openTag = ""; + stream.Seek(0, SeekOrigin.Begin); + + using IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA1); + var buffer = new byte[1 << 20]; + long remaining = checksumTagOffset; + while (remaining > 0) + { + int want = (int)Math.Min(buffer.Length, remaining); + int read = stream.Read(buffer, 0, want); + if (read <= 0) break; + hash.AppendData(buffer, 0, read); + remaining -= read; + } + + hash.AppendData(Encoding.UTF8.GetBytes(openTag)); + return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + } + + private static (long IndexListOffset, string Checksum, long ChecksumTagOffset) ReadTrailer( + FileStream stream, long fileLength) + { + int tailLength = (int)Math.Min(64 * 1024, fileLength); + var tail = new byte[tailLength]; + stream.Seek(fileLength - tailLength, SeekOrigin.Begin); + MzMLFile.ReadExactly(stream, tail, tailLength); + string text = Encoding.UTF8.GetString(tail); + long tailStart = fileLength - tailLength; + + long indexListOffset = -1; + int at = text.LastIndexOf("", StringComparison.Ordinal); + if (at >= 0) + { + int start = at + "".Length; + int end = text.IndexOf("", start, StringComparison.Ordinal); + if (end > start) + { + long.TryParse(text.AsSpan(start, end - start).Trim(), NumberStyles.Integer, + CultureInfo.InvariantCulture, out indexListOffset); + } + } + + string checksum = string.Empty; + long checksumTagOffset = -1; + int checksumAt = text.LastIndexOf("", StringComparison.Ordinal); + if (checksumAt >= 0) + { + checksumTagOffset = tailStart + Encoding.UTF8.GetByteCount(text[..checksumAt]); + int start = checksumAt + "".Length; + int end = text.IndexOf("", start, StringComparison.Ordinal); + if (end > start) checksum = text[start..end].Trim(); + } + + return (indexListOffset, checksum, checksumTagOffset); + } + + /// + /// Reads the index list, which runs from the recorded offset to the end of the file. + /// + /// + /// The offset is read out of the file being validated, so it cannot be trusted to be + /// sane - a truncated or corrupt file is exactly what this method is pointed at. An + /// offset near the start of the file would otherwise have this allocate the whole run: + /// on a 5 GB mzML that is an out-of-memory crash reported as a validator bug, when the + /// finding is that the file's index is broken. Beyond the cap the index is not read and + /// the offsets it would have carried are reported as unvalidated. + /// + private static string ReadIndexList(FileStream stream, long offset, long fileLength) + { + // One offset per spectrum, ~80 bytes each, so this is room for tens of millions - + // orders of magnitude above any real run, and still a bounded allocation. + const long MaxIndexListBytes = 512L * 1024 * 1024; + + if (offset >= fileLength) return string.Empty; + + long available = fileLength - offset; + if (available > MaxIndexListBytes) return string.Empty; + + stream.Seek(offset, SeekOrigin.Begin); + int length = (int)available; + var buffer = new byte[length]; + int total = 0; + while (total < length) + { + int read = stream.Read(buffer, total, length - total); + if (read <= 0) break; + total += read; + } + + return Encoding.UTF8.GetString(buffer, 0, total); + } + + private static List<(string Name, string Id, long Offset)> ParseIndex(string indexXml) + { + var entries = new List<(string, string, long)>(); + string currentName = string.Empty; + int cursor = 0; + + while (true) + { + int indexAt = indexXml.IndexOf("= 0 && (offsetAt < 0 || indexAt < offsetAt)) + { + int start = indexAt + "', idEnd) + 1; + int valueEnd = indexXml.IndexOf('<', valueStart); + if (valueStart <= 0 || valueEnd < 0) break; + + if (long.TryParse(indexXml.AsSpan(valueStart, valueEnd - valueStart), NumberStyles.Integer, + CultureInfo.InvariantCulture, out long offset)) + { + entries.Add((currentName, id, offset)); + } + + cursor = valueEnd; + } + + return entries; + } + + private static string Truncate(string value, int length) + { + value = value.Replace("\n", "\\n").Replace("\r", string.Empty); + return value.Length <= length ? value : value[..length]; + } +} diff --git a/dotnet/MARS.IO/MzMLWriter.cs b/dotnet/MARS.IO/MzMLWriter.cs new file mode 100644 index 0000000..414c846 --- /dev/null +++ b/dotnet/MARS.IO/MzMLWriter.cs @@ -0,0 +1,427 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Byte-splicing mzML writer: copies the input verbatim except for the m/z arrays it is +// asked to replace, then regenerates the index and checksum. + +using System; +using System.Buffers; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using MARS.Core; + +namespace MARS.IO; + +public readonly struct MzTransformResult +{ + /// False leaves the spectrum's bytes untouched. + public bool Rewrite { get; init; } + + public int MonotonicityFixes { get; init; } + + public bool Reverted { get; init; } +} + +/// Per-worker m/z transform. One instance is created per pipeline worker. +public interface IMzTransform +{ + /// Fills with the m/z values to write. + MzTransformResult Transform(SpectrumRecord spectrum, Span corrected); +} + +/// +/// An identity transform: decodes and re-encodes every m/z array without changing a value. +/// This is the null correction the passthrough acceptance test runs, and it is the cheapest +/// way to prove the file-format work is right before any science is layered on top. +/// +public sealed class NullMzTransform : IMzTransform +{ + public MzTransformResult Transform(SpectrumRecord spectrum, Span corrected) + { + spectrum.Mz.CopyTo(corrected); + return new MzTransformResult { Rewrite = true }; + } +} + +public sealed class MzMLWriteOptions +{ + /// + /// Workers used for the per-spectrum decode, predict and re-encode. Inference carries + /// no cross-row accumulation, so parallelizing it cannot change any result - this bounds + /// CPU use, not output. -1 means one per processor. + /// + public int MaxDegreeOfParallelism { get; set; } = -1; + + /// Spectra allowed in flight. Bounds memory; output order is always preserved. + public int MaxPendingSpectra { get; set; } = 512; +} + +public sealed class MzMLWriteResult +{ + public long SpectraSeen { get; init; } + + public long SpectraCorrected { get; init; } + + public long ChromatogramsCopied { get; init; } + + public long MonotonicityFixes { get; init; } + + public long SpectraReverted { get; init; } + + public long OutputLength { get; init; } + + public string FileChecksum { get; init; } = string.Empty; + + public bool WroteIndex { get; init; } +} + +public static class MzMLWriter +{ + /// + /// Writes a corrected copy of to . + /// + public static MzMLWriteResult Write( + MzMLFileInfo info, + string outputPath, + Func workerFactory, + MzMLWriteOptions? options = null, + Action? log = null) + { + options ??= new MzMLWriteOptions(); + int workers = options.MaxDegreeOfParallelism <= 0 + ? Environment.ProcessorCount + : options.MaxDegreeOfParallelism; + + string? directory = Path.GetDirectoryName(Path.GetFullPath(outputPath)); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + + var statePool = new ConcurrentBag(); + + // Caps how many spectra are decoded and scored at once. Without it the work goes + // straight to the thread pool and MaxDegreeOfParallelism - so --threads - decides + // nothing on this path: concurrency would be bounded only by MaxPendingSpectra, two + // orders of magnitude above what the caller asked for. The output is unaffected either + // way, since results are drained in submission order, but a user limiting MARS to one + // core on a shared machine has to actually get one core. + var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, workers); + + using FileStream input = new(info.Path, FileMode.Open, FileAccess.Read, FileShare.Read, + 1 << 20, FileOptions.SequentialScan); + using FileStream output = new(outputPath, FileMode.Create, FileAccess.Write, FileShare.None, + 1 << 20, FileOptions.SequentialScan); + using IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA1); + using var scanner = new MzMLSpanScanner(input, info.ContentCutOffset); + + var spectrumIndex = new List<(string Id, long Offset)>(); + var chromatogramIndex = new List<(string Id, long Offset)>(); + var pending = new Queue(); + + long outputPosition = 0; + long spectraSeen = 0, spectraCorrected = 0, monotonicityFixes = 0, spectraReverted = 0; + + void Emit(byte[] buffer, int start, int count) + { + output.Write(buffer, start, count); + hash.AppendData(buffer, start, count); + outputPosition += count; + } + + void Drain(PendingItem item) + { + if (item.Work is not null) + { + SpectrumOutcome outcome = item.Work.GetAwaiter().GetResult(); + spectrumIndex.Add((outcome.Id, outputPosition)); + if (outcome.Corrected) spectraCorrected++; + monotonicityFixes += outcome.MonotonicityFixes; + if (outcome.Reverted) spectraReverted++; + Emit(outcome.Bytes, 0, outcome.Length); + ArrayPool.Shared.Return(outcome.Bytes); + if (!ReferenceEquals(outcome.Bytes, item.Bytes)) ArrayPool.Shared.Return(item.Bytes!); + return; + } + + if (item.Kind == MzMLRegionKind.Chromatogram) + chromatogramIndex.Add((item.Id ?? string.Empty, outputPosition)); + + Emit(item.Bytes!, 0, item.Length); + ArrayPool.Shared.Return(item.Bytes!); + } + + while (scanner.TryReadRegion(out MzMLRegionKind kind, out int start, out int length)) + { + if (kind == MzMLRegionKind.Spectrum) + { + spectraSeen++; + + byte[] copy = ArrayPool.Shared.Rent(length); + Array.Copy(scanner.Buffer, start, copy, 0, length); + int spanLength = length; + + Task work = Task.Factory.StartNew( + () => ProcessSpectrum(copy, spanLength, info, workerFactory, statePool), + default, TaskCreationOptions.DenyChildAttach, scheduler.ConcurrentScheduler); + + pending.Enqueue(new PendingItem { Work = work, Bytes = copy, Length = spanLength, Kind = kind }); + + while (pending.Count > options.MaxPendingSpectra) Drain(pending.Dequeue()); + } + else if (pending.Count == 0 && kind == MzMLRegionKind.Gap) + { + // Nothing is in flight, so the gap can go straight out. This is the common + // case between spectra, where the gap is a newline and some indentation. + Emit(scanner.Buffer, start, length); + } + else + { + byte[] copy = ArrayPool.Shared.Rent(length); + Array.Copy(scanner.Buffer, start, copy, 0, length); + string? id = kind == MzMLRegionKind.Chromatogram + ? ReadIdAttribute(copy, length) + : null; + pending.Enqueue(new PendingItem { Bytes = copy, Length = length, Kind = kind, Id = id }); + + while (pending.Count > options.MaxPendingSpectra) Drain(pending.Dequeue()); + } + + scanner.Advance(length); + } + + while (pending.Count > 0) Drain(pending.Dequeue()); + + bool writeIndex = info.IsIndexedMzML; + long indexListOffset = outputPosition; + string checksum; + + if (writeIndex) + { + byte[] indexBytes = BuildIndex(spectrumIndex, chromatogramIndex); + Emit(indexBytes, 0, indexBytes.Length); + + byte[] offsetLine = Encoding.UTF8.GetBytes( + " " + indexListOffset.ToString(CultureInfo.InvariantCulture) + "\n"); + Emit(offsetLine, 0, offsetLine.Length); + + // The mzML checksum covers every byte up to AND INCLUDING the fileChecksum + // opening tag. Verified against pwiz-written input, whose recorded digest only + // reproduces under that convention. + byte[] checksumOpen = " "u8.ToArray(); + Emit(checksumOpen, 0, checksumOpen.Length); + + checksum = Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + byte[] tail = Encoding.UTF8.GetBytes(checksum + "\n"); + output.Write(tail, 0, tail.Length); + outputPosition += tail.Length; + } + else + { + checksum = string.Empty; + log?.Invoke("Input is a plain mzML with no indexedmzML wrapper; writing an unindexed copy. " + + "DIA-NN requires an indexed file, so run msconvert with indexing before MARS."); + } + + output.Flush(); + + return new MzMLWriteResult + { + SpectraSeen = spectraSeen, + SpectraCorrected = spectraCorrected, + ChromatogramsCopied = chromatogramIndex.Count, + MonotonicityFixes = monotonicityFixes, + SpectraReverted = spectraReverted, + OutputLength = outputPosition, + FileChecksum = checksum, + WroteIndex = writeIndex, + }; + } + + private sealed class PendingItem + { + public Task? Work; + public byte[]? Bytes; + public int Length; + public MzMLRegionKind Kind; + public string? Id; + } + + private readonly struct SpectrumOutcome + { + public SpectrumOutcome(byte[] bytes, int length, string id, bool corrected, int fixes, bool reverted) + { + Bytes = bytes; + Length = length; + Id = id; + Corrected = corrected; + MonotonicityFixes = fixes; + Reverted = reverted; + } + + public byte[] Bytes { get; } + + public int Length { get; } + + public string Id { get; } + + public bool Corrected { get; } + + public int MonotonicityFixes { get; } + + public bool Reverted { get; } + } + + private sealed class WorkerState + { + public IMzTransform Transform = null!; + public MzMLFile.DecodeContext Decode = new(); + public double[] Corrected = new double[4096]; + public byte[] RawScratch = new byte[1 << 16]; + public byte[] DeflateScratch = new byte[1 << 16]; + public byte[] Base64Scratch = new byte[1 << 16]; + } + + private static SpectrumOutcome ProcessSpectrum( + byte[] span, + int length, + MzMLFileInfo info, + Func workerFactory, + ConcurrentBag statePool) + { + if (!statePool.TryTake(out WorkerState? state)) + state = new WorkerState { Transform = workerFactory() }; + + try + { + ParsedSpectrum parsed = MzMLSpectrumParser.Parse(span, 0, length); + SpectrumRecord record = parsed.Record; + + if (parsed.MzArrayIndex < 0 || !state.Decode.Decode(span, 0, parsed, record)) + return new SpectrumOutcome(span, length, record.Id, false, 0, false); + + record.AcquisitionStartTime = info.AcquisitionStartTime; + record.AbsoluteTime = info.AcquisitionStartTime is double t + ? t + (record.RetentionTime * 60.0) + : record.RetentionTime * 60.0; + + if (state.Corrected.Length < record.PeakCount) + state.Corrected = new double[Math.Max(record.PeakCount, 4096)]; + + var corrected = state.Corrected.AsSpan(0, record.PeakCount); + MzTransformResult result = state.Transform.Transform(record, corrected); + if (!result.Rewrite) + return new SpectrumOutcome(span, length, record.Id, false, result.MonotonicityFixes, result.Reverted); + + BinaryArrayLocation mzArray = parsed.Arrays[parsed.MzArrayIndex]; + if (mzArray.BinaryTextStart < 0 || mzArray.EncodedLengthStart < 0) + return new SpectrumOutcome(span, length, record.Id, false, 0, false); + + int base64Length = MzMLBinaryCodec.Encode( + corrected, mzArray.Encoding, + ref state.RawScratch, ref state.DeflateScratch, ref state.Base64Scratch); + + byte[] replacement = Splice(span, length, mzArray, state.Base64Scratch, base64Length, out int newLength); + return new SpectrumOutcome( + replacement, newLength, record.Id, true, result.MonotonicityFixes, result.Reverted); + } + finally + { + statePool.Add(state); + } + } + + /// + /// Rebuilds the spectrum bytes with a new base64 payload and a matching encodedLength. + /// Everything outside those two ranges is copied byte for byte. + /// + private static byte[] Splice( + byte[] span, + int length, + BinaryArrayLocation mzArray, + byte[] base64, + int base64Length, + out int newLength) + { + // encodedLength records the base64 CHARACTER count, not the decoded byte count. + byte[] lengthText = Encoding.UTF8.GetBytes(base64Length.ToString(CultureInfo.InvariantCulture)); + + int encodedLengthEnd = mzArray.EncodedLengthStart + mzArray.EncodedLengthLength; + int binaryTextEnd = mzArray.BinaryTextStart + mzArray.BinaryTextLength; + + newLength = mzArray.EncodedLengthStart + + lengthText.Length + + (mzArray.BinaryTextStart - encodedLengthEnd) + + base64Length + + (length - binaryTextEnd); + + byte[] result = ArrayPool.Shared.Rent(newLength); + int at = 0; + + Array.Copy(span, 0, result, at, mzArray.EncodedLengthStart); + at += mzArray.EncodedLengthStart; + + Array.Copy(lengthText, 0, result, at, lengthText.Length); + at += lengthText.Length; + + int middle = mzArray.BinaryTextStart - encodedLengthEnd; + Array.Copy(span, encodedLengthEnd, result, at, middle); + at += middle; + + Array.Copy(base64, 0, result, at, base64Length); + at += base64Length; + + int tail = length - binaryTextEnd; + Array.Copy(span, binaryTextEnd, result, at, tail); + + return result; + } + + private static byte[] BuildIndex( + List<(string Id, long Offset)> spectra, + List<(string Id, long Offset)> chromatograms) + { + var builder = new StringBuilder(64 * (spectra.Count + chromatograms.Count) + 256); + builder.Append("\n"); + AppendIndex(builder, "spectrum", spectra); + AppendIndex(builder, "chromatogram", chromatograms); + builder.Append(" \n"); + return Encoding.UTF8.GetBytes(builder.ToString()); + } + + private static void AppendIndex(StringBuilder builder, string name, List<(string Id, long Offset)> entries) + { + builder.Append(" \n"); + foreach ((string id, long offset) in entries) + { + builder.Append(" ") + .Append(offset.ToString(CultureInfo.InvariantCulture)).Append("\n"); + } + + builder.Append(" \n"); + } + + private static string EscapeAttribute(string value) + { + if (value.IndexOfAny(new[] { '&', '<', '>', '"' }) < 0) return value; + return value.Replace("&", "&").Replace("<", "<").Replace(">", ">").Replace("\"", """); + } + + /// Reads the id attribute from an element's opening tag. + private static string ReadIdAttribute(byte[] span, int length) + { + var window = new ReadOnlySpan(span, 0, Math.Min(length, 4096)); + int tagEnd = window.IndexOf((byte)'>'); + if (tagEnd < 0) tagEnd = window.Length; + + ReadOnlySpan attr = " id=\""u8; + int at = window[..tagEnd].IndexOf(attr); + if (at < 0) return string.Empty; + + int start = at + attr.Length; + int end = window[start..tagEnd].IndexOf((byte)'"'); + if (end < 0) return string.Empty; + + return Encoding.UTF8.GetString(span, start, end); + } +} diff --git a/dotnet/MARS.IO/PrismCsvLibraryReader.cs b/dotnet/MARS.IO/PrismCsvLibraryReader.cs new file mode 100644 index 0000000..9f2166d --- /dev/null +++ b/dotnet/MARS.IO/PrismCsvLibraryReader.cs @@ -0,0 +1,294 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from load_prism_library in mars/library.py. + +using System; +using System.Collections.Generic; +using System.IO; +using MARS.Core; + +namespace MARS.IO; + +public sealed class PrismLibraryOptions +{ + /// + /// mzML file names being processed. Rows from other replicates are skipped. Empty means + /// no filtering. + /// + public IReadOnlyList RunNames { get; set; } = Array.Empty(); + + /// Keep modified sequences, for per-match reporting. Costs memory at plate scale. + public bool KeepSequences { get; set; } + + /// + /// Collapse transitions that repeat across replicates. A Skyline report lists every + /// transition once per replicate with an identical theoretical Product Mz, so the copies + /// are exact duplicates: they multiply matching work and training rows without adding + /// information. + /// + public bool DedupeFragments { get; set; } = true; +} + +public static class PrismCsvLibraryReader +{ + public const string PeptideColumn = "Peptide Modified Sequence Unimod Ids"; + public const string PrecursorChargeColumn = "Precursor Charge"; + public const string PrecursorMzColumn = "Precursor Mz"; + public const string FragmentIonColumn = "Fragment Ion"; + public const string ProductChargeColumn = "Product Charge"; + public const string ProductMzColumn = "Product Mz"; + public const string StartTimeColumn = "Start Time"; + public const string EndTimeColumn = "End Time"; + public const string AreaColumn = "Area"; + public const string FileNameColumn = "File Name"; + public const string ReplicateNameColumn = "Replicate Name"; + + public static SpectralLibrary Load(string path, PrismLibraryOptions options, Action? log = null) + { + if (!File.Exists(path)) throw new FileNotFoundException("PRISM CSV not found.", path); + + using var csv = new CsvReader(path); + if (!csv.ReadHeader()) throw new InvalidDataException($"PRISM CSV is empty: {path}"); + + IReadOnlyList missing = csv.RequireColumns( + PeptideColumn, PrecursorChargeColumn, PrecursorMzColumn, + FragmentIonColumn, ProductChargeColumn, ProductMzColumn, + StartTimeColumn, EndTimeColumn); + if (missing.Count > 0) + throw new InvalidDataException($"Missing required columns in {Path.GetFileName(path)}: {string.Join(", ", missing)}"); + + int peptideAt = csv.ColumnIndex(PeptideColumn); + int chargeAt = csv.ColumnIndex(PrecursorChargeColumn); + int precursorMzAt = csv.ColumnIndex(PrecursorMzColumn); + int fragmentIonAt = csv.ColumnIndex(FragmentIonColumn); + int productChargeAt = csv.ColumnIndex(ProductChargeColumn); + int productMzAt = csv.ColumnIndex(ProductMzColumn); + int startTimeAt = csv.ColumnIndex(StartTimeColumn); + int endTimeAt = csv.ColumnIndex(EndTimeColumn); + int areaAt = csv.ColumnIndex(AreaColumn); + + // Skyline writes File Name when it has one, and Replicate Name always. + int filterAt = csv.HasColumn(FileNameColumn) ? csv.ColumnIndex(FileNameColumn) : csv.ColumnIndex(ReplicateNameColumn); + string filterColumn = csv.HasColumn(FileNameColumn) ? FileNameColumn : ReplicateNameColumn; + + var runFilter = new RunNameFilter(options.RunNames); + bool filtering = runFilter.Active && filterAt >= 0; + if (options.RunNames.Count > 0 && !filtering) + log?.Invoke($"PRISM CSV has no {FileNameColumn} or {ReplicateNameColumn} column; using every row."); + + var builder = new SpectralLibraryBuilder(options.KeepSequences, options.DedupeFragments); + var seenKeys = new HashSet<(string, int)>(); + + string currentPeptide = string.Empty; + int currentCharge = int.MinValue; + bool haveEntry = false; + + long rowsRead = 0, rowsFiltered = 0, precursorRows = 0, fragmentRows = 0, duplicateFragments = 0; + long repeatedGroups = 0; + var distinctFilterValues = new HashSet(StringComparer.OrdinalIgnoreCase); + + // A plate-scale Skyline report runs to tens of gigabytes and tens of millions of + // rows. Without a heartbeat the load looks like a hang. + const long ProgressInterval = 5_000_000; + long nextProgress = ProgressInterval; + + while (csv.ReadRow()) + { + rowsRead++; + + if (rowsRead >= nextProgress) + { + log?.Invoke($" {rowsRead:N0} rows read, {builder.EntryCount:N0} precursors so far..."); + nextProgress += ProgressInterval; + } + + if (filtering) + { + string runValue = csv.Field(filterAt); + if (distinctFilterValues.Count < 64) distinctFilterValues.Add(runValue); + if (!runFilter.Matches(runValue)) + { + rowsFiltered++; + continue; + } + } + + string fragmentIon = csv.Field(fragmentIonAt); + if (string.Equals(fragmentIon, "precursor", StringComparison.OrdinalIgnoreCase)) + { + precursorRows++; + continue; + } + + string peptide = csv.Field(peptideAt); + int charge = csv.IntField(chargeAt); + + if (!haveEntry || charge != currentCharge || !string.Equals(peptide, currentPeptide, StringComparison.Ordinal)) + { + if (haveEntry) builder.EndEntry(); + + if (!seenKeys.Add((peptide, charge))) repeatedGroups++; + + builder.BeginEntry( + peptide, + charge, + csv.DoubleField(precursorMzAt), + csv.DoubleField(startTimeAt), + csv.DoubleField(endTimeAt)); + + currentPeptide = peptide; + currentCharge = charge; + haveEntry = true; + } + + double productMz = csv.DoubleField(productMzAt); + if (double.IsNaN(productMz) || productMz <= 0) continue; + + double area = areaAt >= 0 ? csv.DoubleField(areaAt) : 1.0; + if (double.IsNaN(area)) area = 1.0; + + (char ionType, int ionNumber) = ParseFragmentIon(fragmentIon); + int productCharge = csv.IntField(productChargeAt, 1); + + if (builder.AddFragment(productMz, area, ionType, ionNumber, productCharge)) fragmentRows++; + else duplicateFragments++; + } + + if (haveEntry) builder.EndEntry(); + + SpectralLibrary library = builder.Build(); + + log?.Invoke($"PRISM CSV: {rowsRead:N0} rows read, {precursorRows:N0} precursor rows skipped"); + if (filtering) + { + log?.Invoke($" {rowsFiltered:N0} rows skipped by {filterColumn} filter " + + $"({runFilter.Describe()})"); + } + + if (duplicateFragments > 0) + log?.Invoke($" {duplicateFragments:N0} duplicate transitions collapsed across replicates"); + + if (repeatedGroups > 0) + { + log?.Invoke($" WARNING: {repeatedGroups:N0} precursors appear in more than one block. " + + "Each block became its own library entry, which can duplicate matches."); + } + + log?.Invoke($" {library.EntryCount:N0} precursors, {library.FragmentCount:N0} fragments"); + + if (library.EntryCount == 0) + { + string values = string.Join(", ", distinctFilterValues); + throw new InvalidDataException( + filtering + ? $"No PRISM CSV rows matched the input files. Column '{filterColumn}' holds: {values}. " + + $"Inputs normalize to: {runFilter.Describe()}." + : $"No usable fragment rows in {Path.GetFileName(path)}."); + } + + return library; + } + + /// + /// Splits a Skyline fragment annotation such as "y7", "b3", "y5-H2O" into its ion type + /// and series number. Anything unrecognized becomes '?' with number 0, which matches how + /// the Python implementation fills unparsed annotations. + /// + public static (char IonType, int IonNumber) ParseFragmentIon(string annotation) + { + char ionType = '?'; + if (annotation.Length > 0) + { + char first = annotation[0]; + if (first is 'y' or 'b' or 'a' or 'z' or 'c' or 'x') ionType = first; + } + + var ionNumber = 0; + for (int i = 0; i < annotation.Length; i++) + { + if (!char.IsDigit(annotation[i])) continue; + + int j = i; + while (j < annotation.Length && char.IsDigit(annotation[j])) + { + ionNumber = (ionNumber * 10) + (annotation[j] - '0'); + if (ionNumber > short.MaxValue) return (ionType, 0); + j++; + } + + break; + } + + return (ionType, ionNumber); + } +} + +/// +/// Decides whether a replicate or file name in a report belongs to the run being processed. +/// Skyline reports name replicates in whatever way the analyst set up, so matching tries an +/// exact base-name match first and only then falls back to a substring test. +/// +public sealed class RunNameFilter +{ + private static readonly string[] InputSuffixes = { "_uncalibrated", "_calibrated", "-mars", ".mzML", ".mzml", ".raw" }; + private static readonly string[] ReportSuffixes = { ".mzML", ".mzml", ".raw", ".wiff", ".d", "-mars" }; + + private readonly List _baseNames = new(); + private readonly Dictionary _cache = new(StringComparer.OrdinalIgnoreCase); + + public RunNameFilter(IReadOnlyList runNames) + { + foreach (string name in runNames) + { + string baseName = Path.GetFileNameWithoutExtension(name); + foreach (string suffix in InputSuffixes) + baseName = baseName.Replace(suffix, string.Empty, StringComparison.OrdinalIgnoreCase); + if (baseName.Length > 0 && !_baseNames.Contains(baseName)) _baseNames.Add(baseName); + } + } + + public bool Active => _baseNames.Count > 0; + + public string Describe() => string.Join(", ", _baseNames); + + public bool Matches(string reportValue) + { + if (!Active) return true; + if (_cache.TryGetValue(reportValue, out bool cached)) return cached; + + string normalized = reportValue; + foreach (string suffix in ReportSuffixes) + { + if (normalized.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) + { + normalized = normalized[..^suffix.Length]; + break; + } + } + + var matched = false; + foreach (string baseName in _baseNames) + { + if (string.Equals(normalized, baseName, StringComparison.OrdinalIgnoreCase)) + { + matched = true; + break; + } + } + + if (!matched) + { + foreach (string baseName in _baseNames) + { + if (normalized.Contains(baseName, StringComparison.OrdinalIgnoreCase) || + baseName.Contains(normalized, StringComparison.OrdinalIgnoreCase)) + { + matched = true; + break; + } + } + } + + if (_cache.Count < 4096) _cache[reportValue] = matched; + return matched; + } +} diff --git a/dotnet/MARS.IO/Sqlite/SqliteFile.cs b/dotnet/MARS.IO/Sqlite/SqliteFile.cs new file mode 100644 index 0000000..09b3326 --- /dev/null +++ b/dotnet/MARS.IO/Sqlite/SqliteFile.cs @@ -0,0 +1,460 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// +// A read-only SQLite reader: enough of the file format to full-scan a table. +// +// MARS deliberately has no native dependencies, so that the assembly drops into a managed +// ProteoWizard tree without adding per-platform build artifacts. Microsoft.Data.Sqlite +// would pull in SQLitePCLRaw and a native e_sqlite3 for every runtime identifier, which is +// exactly the thing being avoided. BiblioSpec libraries only ever need sequential scans of +// a handful of tables, and that is a small, well-specified subset of the format. +// +// Supports: table b-trees (interior and leaf), overflow page chains, the record format, +// and UTF-8 / UTF-16 text. Does NOT support: indices, WAL, encryption, or writing. + +using System; +using System.Globalization; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace MARS.IO.Sqlite; + +public enum SqliteValueKind +{ + Null, + Integer, + Real, + Text, + Blob, +} + +/// One column value from a row. +public readonly struct SqliteValue +{ + private readonly long _integer; + private readonly double _real; + private readonly byte[]? _bytes; + private readonly int _offset; + private readonly int _length; + private readonly Encoding? _encoding; + + private SqliteValue(SqliteValueKind kind, long integer, double real, byte[]? bytes, int offset, int length, Encoding? encoding) + { + Kind = kind; + _integer = integer; + _real = real; + _bytes = bytes; + _offset = offset; + _length = length; + _encoding = encoding; + } + + public SqliteValueKind Kind { get; } + + public bool IsNull => Kind == SqliteValueKind.Null; + + public static SqliteValue Null() => new(SqliteValueKind.Null, 0, 0, null, 0, 0, null); + + public static SqliteValue FromInteger(long value) => new(SqliteValueKind.Integer, value, 0, null, 0, 0, null); + + public static SqliteValue FromReal(double value) => new(SqliteValueKind.Real, 0, value, null, 0, 0, null); + + public static SqliteValue FromText(byte[] bytes, int offset, int length, Encoding encoding) => + new(SqliteValueKind.Text, 0, 0, bytes, offset, length, encoding); + + public static SqliteValue FromBlob(byte[] bytes, int offset, int length) => + new(SqliteValueKind.Blob, 0, 0, bytes, offset, length, null); + + public long AsInteger() => Kind switch + { + SqliteValueKind.Integer => _integer, + SqliteValueKind.Real => (long)_real, + // Invariant, explicitly. A SQLite text value holds a number the way SQLite wrote it, + // which has nothing to do with the locale of the machine reading it - a library built + // in Seattle has to read the same in Munich. + SqliteValueKind.Text => + long.TryParse(AsText(), NumberStyles.Integer, CultureInfo.InvariantCulture, out long parsed) + ? parsed + : 0, + _ => 0, + }; + + public double AsDouble() => Kind switch + { + SqliteValueKind.Real => _real, + SqliteValueKind.Integer => _integer, + SqliteValueKind.Text => + double.TryParse(AsText(), NumberStyles.Float, CultureInfo.InvariantCulture, out double parsed) + ? parsed + : double.NaN, + _ => double.NaN, + }; + + public string AsText() => Kind switch + { + SqliteValueKind.Text => _encoding!.GetString(_bytes!, _offset, _length), + SqliteValueKind.Blob => Encoding.UTF8.GetString(_bytes!, _offset, _length), + SqliteValueKind.Integer => _integer.ToString(System.Globalization.CultureInfo.InvariantCulture), + SqliteValueKind.Real => _real.ToString(System.Globalization.CultureInfo.InvariantCulture), + _ => string.Empty, + }; + + /// The raw bytes of a blob. The array is the row's payload buffer; copy to keep. + public ReadOnlySpan AsBlob() => + _bytes is null ? ReadOnlySpan.Empty : new ReadOnlySpan(_bytes, _offset, _length); +} + +/// A table's schema entry from sqlite_master. +public sealed class SqliteTable +{ + public required string Name { get; init; } + + public required int RootPage { get; init; } + + public required string CreateSql { get; init; } + + public required IReadOnlyList Columns { get; init; } + + public int ColumnIndex(string name) + { + for (int i = 0; i < Columns.Count; i++) + { + if (string.Equals(Columns[i], name, StringComparison.OrdinalIgnoreCase)) return i; + } + + return -1; + } +} + +public sealed class SqliteFile : IDisposable +{ + private readonly FileStream _stream; + private readonly int _pageSize; + private readonly int _usableSize; + private readonly Encoding _textEncoding; + private readonly byte[] _page; + private readonly Dictionary _tables = new(StringComparer.OrdinalIgnoreCase); + + public SqliteFile(string path) + { + _stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16, FileOptions.RandomAccess); + + var header = new byte[100]; + MzMLFile.ReadExactly(_stream, header, header.Length); + + if (Encoding.ASCII.GetString(header, 0, 15) != "SQLite format 3") + throw new InvalidDataException($"Not a SQLite database: {path}"); + + int declaredPageSize = (header[16] << 8) | header[17]; + _pageSize = declaredPageSize == 1 ? 65536 : declaredPageSize; + if (_pageSize < 512 || (_pageSize & (_pageSize - 1)) != 0) + throw new InvalidDataException($"Invalid SQLite page size {_pageSize} in {path}"); + + int reserved = header[20]; + _usableSize = _pageSize - reserved; + + int encoding = ReadInt32BigEndian(header, 56); + _textEncoding = encoding switch + { + 2 => Encoding.Unicode, + 3 => Encoding.BigEndianUnicode, + _ => Encoding.UTF8, + }; + + _page = new byte[_pageSize]; + LoadSchema(); + } + + public IReadOnlyDictionary Tables => _tables; + + public bool HasTable(string name) => _tables.ContainsKey(name); + + public SqliteTable Table(string name) => + _tables.TryGetValue(name, out SqliteTable? table) + ? table + : throw new InvalidDataException($"Table '{name}' not found in the database."); + + /// + /// Streams every row of a table in b-tree order. The values reference a shared payload + /// buffer that is reused between rows, so blobs must be copied to outlive the iteration. + /// + public IEnumerable Scan(SqliteTable table) + { + var payload = new byte[_pageSize * 2]; + var values = new SqliteValue[Math.Max(table.Columns.Count, 8)]; + var pages = new Stack(); + pages.Push(table.RootPage); + + // The b-tree is walked with an explicit stack rather than recursion so that a deep + // or corrupt tree cannot blow the call stack. + var pageBuffer = new byte[_pageSize]; + while (pages.Count > 0) + { + int pageNumber = pages.Pop(); + ReadPage(pageNumber, pageBuffer); + int headerOffset = pageNumber == 1 ? 100 : 0; + byte pageType = pageBuffer[headerOffset]; + + int cellCount = (pageBuffer[headerOffset + 3] << 8) | pageBuffer[headerOffset + 4]; + + if (pageType == 0x05) + { + // Interior table page: push children in reverse so they are visited in order. + int rightMost = ReadInt32BigEndian(pageBuffer, headerOffset + 8); + var children = new List(cellCount + 1); + for (int i = 0; i < cellCount; i++) + { + int cellPointer = ReadCellPointer(pageBuffer, headerOffset, i, interior: true); + children.Add(ReadInt32BigEndian(pageBuffer, cellPointer)); + } + + children.Add(rightMost); + for (int i = children.Count - 1; i >= 0; i--) pages.Push(children[i]); + continue; + } + + if (pageType != 0x0d) + { + // Index pages carry no table rows; a table b-tree should never contain them. + continue; + } + + for (int i = 0; i < cellCount; i++) + { + int cellPointer = ReadCellPointer(pageBuffer, headerOffset, i, interior: false); + int at = cellPointer; + + long payloadSize = ReadVarint(pageBuffer, ref at); + long rowId = ReadVarint(pageBuffer, ref at); + + int payloadLength = ReadPayload(pageBuffer, at, payloadSize, ref payload); + int count = ParseRecord(payload, payloadLength, ref values); + yield return new SqliteRow(rowId, values, count); + } + } + } + + public void Dispose() => _stream.Dispose(); + + private void LoadSchema() + { + // sqlite_master always lives at page 1 and has the fixed shape + // (type, name, tbl_name, rootpage, sql). + var master = new SqliteTable + { + Name = "sqlite_master", + RootPage = 1, + CreateSql = string.Empty, + Columns = new[] { "type", "name", "tbl_name", "rootpage", "sql" }, + }; + + foreach (SqliteRow row in Scan(master)) + { + if (row.Count < 5) continue; + if (!string.Equals(row[0].AsText(), "table", StringComparison.OrdinalIgnoreCase)) continue; + + string name = row[1].AsText(); + var rootPage = (int)row[3].AsInteger(); + string sql = row[4].AsText(); + if (rootPage <= 0) continue; + + _tables[name] = new SqliteTable + { + Name = name, + RootPage = rootPage, + CreateSql = sql, + Columns = SqliteSchemaParser.ParseColumns(sql), + }; + } + } + + private void ReadPage(int pageNumber, byte[] destination) + { + long offset = (long)(pageNumber - 1) * _pageSize; + _stream.Seek(offset, SeekOrigin.Begin); + MzMLFile.ReadExactly(_stream, destination, _pageSize); + } + + private static int ReadCellPointer(byte[] page, int headerOffset, int index, bool interior) + { + int arrayStart = headerOffset + (interior ? 12 : 8); + int at = arrayStart + (index * 2); + return (page[at] << 8) | page[at + 1]; + } + + /// + /// Copies a cell's payload into , following the overflow page + /// chain when the record does not fit on its page. + /// + private int ReadPayload(byte[] page, int at, long payloadSize, ref byte[] payload) + { + if (payloadSize > int.MaxValue) throw new InvalidDataException("SQLite payload too large."); + var total = (int)payloadSize; + if (payload.Length < total) payload = new byte[Math.Max(total, payload.Length * 2)]; + + // Table leaf spill thresholds, straight from the file format definition. + int maxLocal = _usableSize - 35; + int minLocal = (((_usableSize - 12) * 32 / 255) - 23); + + int localSize; + if (total <= maxLocal) + { + localSize = total; + } + else + { + int candidate = minLocal + ((total - minLocal) % (_usableSize - 4)); + localSize = candidate > maxLocal ? minLocal : candidate; + } + + Array.Copy(page, at, payload, 0, localSize); + int written = localSize; + + if (written < total) + { + int overflowPage = ReadInt32BigEndian(page, at + localSize); + var overflowBuffer = new byte[_pageSize]; + while (overflowPage != 0 && written < total) + { + ReadPage(overflowPage, overflowBuffer); + int chunk = Math.Min(_usableSize - 4, total - written); + Array.Copy(overflowBuffer, 4, payload, written, chunk); + written += chunk; + overflowPage = ReadInt32BigEndian(overflowBuffer, 0); + } + + if (written < total) + throw new InvalidDataException("SQLite overflow chain ended before the payload was complete."); + } + + return total; + } + + /// Splits a record payload into column values. + private int ParseRecord(byte[] payload, int length, ref SqliteValue[] values) + { + var at = 0; + long headerSize = ReadVarint(payload, ref at); + int headerEnd = (int)headerSize; + int body = headerEnd; + + var count = 0; + while (at < headerEnd && body <= length) + { + long serialType = ReadVarint(payload, ref at); + if (count == values.Length) Array.Resize(ref values, values.Length * 2); + + switch (serialType) + { + case 0: + values[count++] = SqliteValue.Null(); + break; + case 1: + values[count++] = SqliteValue.FromInteger((sbyte)payload[body]); + body += 1; + break; + case 2: + values[count++] = SqliteValue.FromInteger((short)((payload[body] << 8) | payload[body + 1])); + body += 2; + break; + case 3: + { + int raw = (payload[body] << 16) | (payload[body + 1] << 8) | payload[body + 2]; + if ((raw & 0x800000) != 0) raw |= unchecked((int)0xFF000000); + values[count++] = SqliteValue.FromInteger(raw); + body += 3; + break; + } + + case 4: + values[count++] = SqliteValue.FromInteger(ReadInt32BigEndian(payload, body)); + body += 4; + break; + case 5: + { + long raw = 0; + for (var i = 0; i < 6; i++) raw = (raw << 8) | payload[body + i]; + if ((raw & 0x800000000000L) != 0) raw |= unchecked((long)0xFFFF000000000000); + values[count++] = SqliteValue.FromInteger(raw); + body += 6; + break; + } + + case 6: + values[count++] = SqliteValue.FromInteger(ReadInt64BigEndian(payload, body)); + body += 8; + break; + case 7: + values[count++] = SqliteValue.FromReal(BitConverter.Int64BitsToDouble(ReadInt64BigEndian(payload, body))); + body += 8; + break; + case 8: + values[count++] = SqliteValue.FromInteger(0); + break; + case 9: + values[count++] = SqliteValue.FromInteger(1); + break; + case 10: + case 11: + values[count++] = SqliteValue.Null(); + break; + default: + { + var size = (int)((serialType - (serialType % 2 == 0 ? 12 : 13)) / 2); + values[count++] = serialType % 2 == 0 + ? SqliteValue.FromBlob(payload, body, size) + : SqliteValue.FromText(payload, body, size, _textEncoding); + body += size; + break; + } + } + } + + return count; + } + + internal static long ReadVarint(byte[] buffer, ref int offset) + { + long value = 0; + for (var i = 0; i < 8; i++) + { + byte b = buffer[offset++]; + value = (value << 7) | (byte)(b & 0x7F); + if ((b & 0x80) == 0) return value; + } + + // Ninth byte contributes all eight bits. + value = (value << 8) | buffer[offset++]; + return value; + } + + internal static int ReadInt32BigEndian(byte[] buffer, int offset) => + (buffer[offset] << 24) | (buffer[offset + 1] << 16) | (buffer[offset + 2] << 8) | buffer[offset + 3]; + + internal static long ReadInt64BigEndian(byte[] buffer, int offset) + { + long value = 0; + for (var i = 0; i < 8; i++) value = (value << 8) | buffer[offset + i]; + return value; + } +} + +/// One row, valid until the next iteration step. +public readonly struct SqliteRow +{ + private readonly SqliteValue[] _values; + + internal SqliteRow(long rowId, SqliteValue[] values, int count) + { + RowId = rowId; + _values = values; + Count = count; + } + + public long RowId { get; } + + public int Count { get; } + + public SqliteValue this[int index] => + index >= 0 && index < Count ? _values[index] : SqliteValue.Null(); +} diff --git a/dotnet/MARS.IO/Sqlite/SqliteSchemaParser.cs b/dotnet/MARS.IO/Sqlite/SqliteSchemaParser.cs new file mode 100644 index 0000000..2f4559e --- /dev/null +++ b/dotnet/MARS.IO/Sqlite/SqliteSchemaParser.cs @@ -0,0 +1,230 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Pulls column names out of a CREATE TABLE statement. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace MARS.IO.Sqlite; + +/// +/// Recovers the column order of a table from its stored CREATE TABLE text. +/// +/// SQLite keeps no separate column catalog, so the DDL is the only record of order, and the +/// record format is positional. BiblioSpec's schema is heavily commented, so line comments +/// and string literals both have to be stripped before splitting on commas. +/// +/// +public static class SqliteSchemaParser +{ + private static readonly string[] ConstraintKeywords = + { + "PRIMARY", "FOREIGN", "UNIQUE", "CHECK", "CONSTRAINT", + }; + + public static IReadOnlyList ParseColumns(string createSql) + { + if (string.IsNullOrWhiteSpace(createSql)) return Array.Empty(); + + string sql = StripComments(createSql); + + int open = sql.IndexOf('('); + if (open < 0) return Array.Empty(); + + int close = FindMatchingParen(sql, open); + if (close < 0) return Array.Empty(); + + string body = sql[(open + 1)..close]; + var columns = new List(); + + foreach (string part in SplitTopLevel(body)) + { + string definition = part.Trim(); + if (definition.Length == 0) continue; + + string first = FirstToken(definition); + if (first.Length == 0) continue; + + var isConstraint = false; + foreach (string keyword in ConstraintKeywords) + { + if (string.Equals(first, keyword, StringComparison.OrdinalIgnoreCase)) + { + isConstraint = true; + break; + } + } + + if (isConstraint) continue; + columns.Add(Unquote(first)); + } + + return columns; + } + + private static string StripComments(string sql) + { + var text = new StringBuilder(sql.Length); + var inString = false; + char stringQuote = '\0'; + + for (var i = 0; i < sql.Length; i++) + { + char c = sql[i]; + + if (inString) + { + text.Append(c); + if (c == stringQuote) inString = false; + continue; + } + + if (c is '\'' or '"' or '`' or '[') + { + inString = true; + stringQuote = c == '[' ? ']' : c; + text.Append(c); + continue; + } + + if (c == '-' && i + 1 < sql.Length && sql[i + 1] == '-') + { + while (i < sql.Length && sql[i] != '\n') i++; + text.Append('\n'); + continue; + } + + if (c == '/' && i + 1 < sql.Length && sql[i + 1] == '*') + { + i += 2; + while (i + 1 < sql.Length && !(sql[i] == '*' && sql[i + 1] == '/')) i++; + i++; + text.Append(' '); + continue; + } + + text.Append(c); + } + + return text.ToString(); + } + + private static int FindMatchingParen(string sql, int open) + { + var depth = 0; + var inString = false; + char stringQuote = '\0'; + + for (int i = open; i < sql.Length; i++) + { + char c = sql[i]; + + if (inString) + { + if (c == stringQuote) inString = false; + continue; + } + + switch (c) + { + case '\'': + case '"': + case '`': + inString = true; + stringQuote = c; + break; + case '[': + inString = true; + stringQuote = ']'; + break; + case '(': + depth++; + break; + case ')': + depth--; + if (depth == 0) return i; + break; + } + } + + return -1; + } + + private static IEnumerable SplitTopLevel(string body) + { + var depth = 0; + var start = 0; + var inString = false; + char stringQuote = '\0'; + + for (var i = 0; i < body.Length; i++) + { + char c = body[i]; + + if (inString) + { + if (c == stringQuote) inString = false; + continue; + } + + switch (c) + { + case '\'': + case '"': + case '`': + inString = true; + stringQuote = c; + break; + case '[': + inString = true; + stringQuote = ']'; + break; + case '(': + depth++; + break; + case ')': + depth--; + break; + case ',' when depth == 0: + yield return body[start..i]; + start = i + 1; + break; + } + } + + if (start < body.Length) yield return body[start..]; + } + + private static string FirstToken(string definition) + { + var i = 0; + while (i < definition.Length && char.IsWhiteSpace(definition[i])) i++; + if (i >= definition.Length) return string.Empty; + + if (definition[i] is '"' or '`' or '[') + { + char closing = definition[i] == '[' ? ']' : definition[i]; + int end = definition.IndexOf(closing, i + 1); + return end < 0 ? definition[i..] : definition[i..(end + 1)]; + } + + int start = i; + while (i < definition.Length && !char.IsWhiteSpace(definition[i]) && + definition[i] != '(' && definition[i] != ',') + { + i++; + } + + return definition[start..i]; + } + + private static string Unquote(string token) + { + if (token.Length < 2) return token; + char first = token[0]; + char last = token[^1]; + if ((first == '"' && last == '"') || (first == '`' && last == '`') || (first == '[' && last == ']')) + return token[1..^1]; + return token; + } +} diff --git a/dotnet/MARS.IO/TemperatureCsvReader.cs b/dotnet/MARS.IO/TemperatureCsvReader.cs new file mode 100644 index 0000000..8a2fa9e --- /dev/null +++ b/dotnet/MARS.IO/TemperatureCsvReader.cs @@ -0,0 +1,121 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from mars/temperature.py. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text.RegularExpressions; +using MARS.Core; + +namespace MARS.IO; + +/// +/// Loads RF-generator temperature traces exported from Xcalibur as chromatogram CSVs. +/// The export carries a few preamble lines before the "Time(min),..." header. +/// +public static class TemperatureCsvReader +{ + private static readonly Regex SourcePattern = new(@"(RF[AC]\d+)", RegexOptions.Compiled); + + public static readonly string[] DefaultSources = { "RFA2", "RFC2" }; + + public static TemperatureData? Load(string path, Action? log = null) + { + if (!File.Exists(path)) return null; + + try + { + string[] lines = File.ReadAllLines(path); + + var headerAt = -1; + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].StartsWith("Time", StringComparison.Ordinal)) + { + headerAt = i; + break; + } + } + + if (headerAt < 0) + { + log?.Invoke($"No Time header in {Path.GetFileName(path)}; skipping."); + return null; + } + + string[] headerFields = lines[headerAt].Split(','); + if (headerFields.Length < 2) + { + log?.Invoke($"Unexpected header in {Path.GetFileName(path)}; skipping."); + return null; + } + + Match sourceMatch = SourcePattern.Match(headerFields[1]); + string source = sourceMatch.Success + ? sourceMatch.Groups[1].Value + : Path.GetFileNameWithoutExtension(path).Split('-')[0]; + + var times = new List(lines.Length - headerAt); + var temperatures = new List(lines.Length - headerAt); + + for (int i = headerAt + 1; i < lines.Length; i++) + { + string line = lines[i]; + if (line.Length == 0) continue; + + int comma = line.IndexOf(','); + if (comma <= 0) continue; + + if (!double.TryParse(line.AsSpan(0, comma), NumberStyles.Float, CultureInfo.InvariantCulture, + out double time)) + { + continue; + } + + ReadOnlySpan rest = line.AsSpan(comma + 1); + int nextComma = rest.IndexOf(','); + if (nextComma >= 0) rest = rest[..nextComma]; + + if (!double.TryParse(rest, NumberStyles.Float, CultureInfo.InvariantCulture, out double temperature)) + continue; + + times.Add(time); + temperatures.Add(temperature); + } + + if (times.Count == 0) + { + log?.Invoke($"No temperature points in {Path.GetFileName(path)}; skipping."); + return null; + } + + var data = new TemperatureData(times.ToArray(), temperatures.ToArray(), source); + log?.Invoke($"Loaded {data.Count:N0} temperature points from {Path.GetFileName(path)} " + + $"(source {source}, {data.MinTemperature:F1} to {data.MaxTemperature:F1} C)"); + return data; + } + catch (IOException ex) + { + log?.Invoke($"Could not read {Path.GetFileName(path)}: {ex.Message}"); + return null; + } + } + + /// + /// Finds the traces belonging to one run. Files are named {source}-{mzml base name}.csv, + /// for example RFA2-Ste-2024-12-02_HeLa_20msIIT_GPFDIA_400-500_14.csv. + /// + public static TemperatureSet Find(string mzmlPath, string? temperatureDirectory, Action? log = null) + { + string baseName = Path.GetFileNameWithoutExtension(mzmlPath); + string directory = string.IsNullOrEmpty(temperatureDirectory) + ? Path.GetDirectoryName(Path.GetFullPath(mzmlPath)) ?? "." + : temperatureDirectory; + + TemperatureData? rfa2 = Load(Path.Combine(directory, $"RFA2-{baseName}.csv"), log); + TemperatureData? rfc2 = Load(Path.Combine(directory, $"RFC2-{baseName}.csv"), log); + + return new TemperatureSet { Rfa2 = rfa2, Rfc2 = rfc2 }; + } +} diff --git a/dotnet/MARS.OspreyML/MARS.OspreyML.csproj b/dotnet/MARS.OspreyML/MARS.OspreyML.csproj new file mode 100644 index 0000000..1b8a83d --- /dev/null +++ b/dotnet/MARS.OspreyML/MARS.OspreyML.csproj @@ -0,0 +1,31 @@ + + + + + pwiz.Osprey.ML + MARS.OspreyML + Vendored Osprey.ML gradient boosted trees. + + disable + false + + + + + + + + diff --git a/dotnet/MARS.Pwiz/MARS.Pwiz.csproj b/dotnet/MARS.Pwiz/MARS.Pwiz.csproj new file mode 100644 index 0000000..77d07c0 --- /dev/null +++ b/dotnet/MARS.Pwiz/MARS.Pwiz.csproj @@ -0,0 +1,117 @@ + + + + + MARS.Pwiz + MARS.Pwiz + Reads vendor formats and writes non-mzML formats, through pwiz-sharp. + + + + + + + $(DefineConstants);MARS_NO_PWIZ + + + + $(DefineConstants);MARS_SCIEX + + + + + + + + + + + + + + + + + + + + + + + + + false + Content + PreserveNewest + + + + false + Content + PreserveNewest + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/MARS.Pwiz/MarsSpectrumList.cs b/dotnet/MARS.Pwiz/MarsSpectrumList.cs new file mode 100644 index 0000000..816a34f --- /dev/null +++ b/dotnet/MARS.Pwiz/MarsSpectrumList.cs @@ -0,0 +1,428 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using MARS.Core; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.Common.Params; +using Pwiz.Data.MsData.Processing; +using Pwiz.Data.MsData.Spectra; + +namespace MARS.Pwiz; + +/// +/// A pwiz spectrum list that applies MARS's correction as spectra are pulled through it. +/// +/// +/// +/// This is the whole of the integration. MARS's science is untouched and reached through the +/// same the byte-splice writer uses, so whichever format is +/// being written gets exactly the values mzML would have got - verified by writing both and +/// diffing them with mars compare, which found no difference across 82,349,582 peaks. +/// +/// +/// Derived from SpectrumListBase in Pwiz.Data.MsData rather than +/// SpectrumListWrapper in Pwiz.Analysis, at the cost of three delegating +/// members. Analysis references the Waters reader, which stages a native Windows-only +/// MassLynxRaw.dll into the output, and MARS ships four non-Windows artifacts. Nothing here +/// needs Analysis. +/// +/// +/// Batched, because the model is where the time goes. Writing one Astral run took 308 s, +/// of which the reader was 16.6 s and pwiz's encoder about 49 s - the remaining 243 s, 79% of +/// it, was scoring the model. pwiz's writers pull spectra one at a time, so left alone that +/// 243 s runs on one core. Instead of serving each pull straight through, this reads a batch +/// ahead and corrects the batch in parallel, then serves the batch one spectrum at a time as +/// the writer asks for them. +/// +/// +/// Reads stay sequential and single-threaded: they are 5% of the cost, and the vendor readers +/// are not thread-safe. Only the correction is parallel, and it is embarrassingly so - each +/// spectrum is independent, and holds nothing mutable, taking +/// its scratch space as an argument. Results do not depend on scheduling, so the output is the +/// same however many threads run. +/// +/// +internal sealed class MarsSpectrumList : SpectrumListBase +{ + /// + /// Spectra read ahead per thread. Enough that the parallel loop is not dominated by its + /// own fan-out cost, small enough that the batch stays a few megabytes: an Astral MS2 is + /// around 2,400 peaks, so two arrays of doubles is roughly 38 KB. + /// + private const int BatchPerThread = 4; + + private const int MaxBatch = 256; + + private readonly ISpectrumList _inner; + private readonly IVendorCentroidingSpectrumList? _centroider; + private readonly SpectrumCorrector? _corrector; + private readonly TemperatureSet? _temperatures; + private readonly double? _acquisitionStart; + private readonly int _threads; + + private readonly Worker _serial = new(); + private readonly Spectrum?[] _batch; + private int _batchStart = -1; + private int _batchCount; + + private long _correctorTicks; + private long _readerTicks; + + public MarsSpectrumList( + ISpectrumList inner, MzCalibrator? calibrator, CorrectionOptions options, + double? acquisitionStart, TemperatureSet? temperatures, int threads) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _centroider = inner as IVendorCentroidingSpectrumList; + _corrector = calibrator is null ? null : new SpectrumCorrector(calibrator, options); + _acquisitionStart = acquisitionStart; + _temperatures = temperatures; + _threads = threads <= 0 ? Environment.ProcessorCount : threads; + _batch = new Spectrum?[Math.Min(MaxBatch, Math.Max(1, _threads) * BatchPerThread)]; + } + + public long SpectraSeen { get; private set; } + + public long SpectraCorrected { get; private set; } + + public long MonotonicityFixes { get; private set; } + + public long SpectraReverted { get; private set; } + + /// Wall time spent inside the reader, pulling spectra from the file. + public TimeSpan ReaderTime => TimeSpan.FromTicks(Interlocked.Read(ref _readerTicks)); + + /// + /// Time spent applying the model, summed across threads - so it is CPU time rather than + /// wall time, and on a parallel run it exceeds the elapsed time of the write. + /// + public TimeSpan CorrectorTime => TimeSpan.FromTicks(Interlocked.Read(ref _correctorTicks)); + + public override int Count => _inner.Count; + + public override SpectrumIdentity SpectrumIdentity(int index) => _inner.SpectrumIdentity(index); + + public override DataProcessing? DataProcessing => _inner.DataProcessing; + + public override Spectrum GetSpectrum(int index, bool getBinaryData = false) + { + // Metadata-only pulls skip the batch entirely: there is nothing to correct, and + // priming a batch of full spectra to answer one would be pure waste. + if (_corrector is null || !getBinaryData || _threads <= 1) return Sequential(index, getBinaryData); + + int slot = index - _batchStart; + bool inBatch = _batchStart >= 0 && slot >= 0 && slot < _batchCount; + + // Corrected, still held, not yet handed over. + if (inBatch && _batch[slot] is not null) return Take(index); + + // In the batch but already handed over: the caller is re-reading a spectrum. Nothing + // here promises single use - a consumer may look at a spectrum twice - so read it + // again rather than refilling the batch around it or, worse, failing. + if (inBatch) return Sequential(index, getBinaryData); + + // Outside the batch: start a new one here. That covers both the ordinary sequential + // walk and a jump - a caller that skips ahead once and then walks in order, which is + // what a writer filtering on MS level looks like, gets the read-ahead back rather + // than losing it for the rest of the file. + FillBatch(index); + return _batchCount > 0 && index >= _batchStart && index < _batchStart + _batchCount + ? Take(index) + : Sequential(index, getBinaryData); + } + + /// + /// Reads one spectrum, centroided by the vendor when the run is stored as profile. + /// + /// + /// The reader centroids too, and both have to, or the model would be fitted on peak lists + /// and applied to sampled curves. The features it is fitted on - fourteen of them counted + /// from the peaks around a match - mean nothing on profile samples of the same ion, so + /// training on one and correcting the other would put the model far outside anything it + /// had seen. + /// + private Spectrum Read(int index) + { + Spectrum spectrum = _inner.GetSpectrum(index, getBinaryData: true); + if (_centroider is null) return spectrum; + + return spectrum.Params.HasCVParam(CVID.MS_profile_spectrum) + ? _centroider.GetCentroidSpectrum(index, getBinaryData: true) + : spectrum; + } + + protected override void DisposeCore() => _inner.Dispose(); + + /// Hands over a spectrum the batch already corrected, and releases the slot. + private Spectrum Take(int index) + { + int slot = index - _batchStart; + Spectrum spectrum = _batch[slot] + ?? throw new InvalidOperationException( + $"Spectrum {index} is no longer held by the batch. GetSpectrum " + + "checks the slot before calling this, so reaching here means the " + + "batch bookkeeping is wrong."); + + // Dropped as soon as it is handed over: the writer encodes it and moves on, and + // holding the whole batch alive until the next refill would double the footprint. + _batch[slot] = null; + return spectrum; + } + + /// + /// Reads the next batch sequentially, then corrects it in parallel. + /// + /// + /// Reading and correcting are separate phases rather than a pipeline. Overlapping them + /// would save the read, which is 5% of the work, at the cost of a producer/consumer queue + /// and the ordering it has to preserve. Two phases is a great deal easier to be sure of. + /// + private void FillBatch(int start) + { + Array.Clear(_batch); + _batchStart = start; + _batchCount = Math.Min(_batch.Length, Count - start); + if (_batchCount <= 0) return; + + long readStart = System.Diagnostics.Stopwatch.GetTimestamp(); + for (int i = 0; i < _batchCount; i++) + _batch[i] = Read(start + i); + AddElapsed(ref _readerTicks, readStart); + + long seen = 0, corrected = 0, fixes = 0, reverted = 0; + + Parallel.For( + 0, + _batchCount, + new ParallelOptions { MaxDegreeOfParallelism = _threads }, + () => new Worker(), + (i, _, worker) => + { + Spectrum? spectrum = _batch[i]; + if (spectrum is not null) Correct(spectrum, worker); + return worker; + }, + worker => + { + // Totals are summed here rather than incremented per spectrum, so the counters + // need no interlocking in the hot loop. + Interlocked.Add(ref seen, worker.Seen); + Interlocked.Add(ref corrected, worker.Corrected); + Interlocked.Add(ref fixes, worker.MonotonicityFixes); + Interlocked.Add(ref reverted, worker.Reverted); + Interlocked.Add(ref _correctorTicks, worker.Ticks); + }); + + SpectraSeen += seen; + SpectraCorrected += corrected; + MonotonicityFixes += fixes; + SpectraReverted += reverted; + } + + /// The unbatched path: read one spectrum and correct it on this thread. + private Spectrum Sequential(int index, bool getBinaryData) + { + long readStart = System.Diagnostics.Stopwatch.GetTimestamp(); + Spectrum spectrum = getBinaryData ? Read(index) : _inner.GetSpectrum(index, false); + AddElapsed(ref _readerTicks, readStart); + + if (_corrector is null || !getBinaryData) return spectrum; + + Correct(spectrum, _serial); + Drain(_serial); + return spectrum; + } + + /// + /// Folds a worker's counters into the totals and zeroes it. + /// + /// + /// Both paths have to accumulate. This one assigned the running totals instead, which was + /// harmless only while the batched and unbatched paths never ran on the same list - so a + /// single spectrum served outside the batch, which a re-read now is, would have reset the + /// file's counts to one. + /// + private void Drain(Worker worker) + { + SpectraSeen += worker.Seen; + SpectraCorrected += worker.Corrected; + MonotonicityFixes += worker.MonotonicityFixes; + SpectraReverted += worker.Reverted; + Interlocked.Add(ref _correctorTicks, worker.Ticks); + + worker.Seen = 0; + worker.Corrected = 0; + worker.MonotonicityFixes = 0; + worker.Reverted = 0; + worker.Ticks = 0; + } + + /// + /// Applies the model to one spectrum, in place, using this worker's scratch space. + /// + private void Correct(Spectrum spectrum, Worker worker) + { + if (spectrum.Params.CvParamValueOrDefault(CVID.MS_ms_level, 0) != 2) return; + + BinaryDataArray? mz = spectrum.GetMZArray(); + BinaryDataArray? intensity = spectrum.GetIntensityArray(); + if (mz is null || intensity is null || mz.Data.Count == 0) return; + + worker.Seen++; + Fill(worker.Record, spectrum, mz, intensity); + + int peaks = mz.Data.Count; + if (worker.Corrections.Length < peaks) worker.Corrections = new double[peaks]; + Span corrected = worker.Corrections.AsSpan(0, peaks); + + // Corrected into scratch rather than in place: the corrector reverts a whole spectrum + // when the correction would reorder its peaks, and it cannot revert what it has + // already overwritten. + long start = System.Diagnostics.Stopwatch.GetTimestamp(); + SpectrumCorrectionResult result = + _corrector!.Correct(worker.Record, _temperatures, worker.Workspace, corrected); + worker.Ticks += Elapsed(start); + + worker.MonotonicityFixes += result.MonotonicityFixes; + if (result.Reverted) worker.Reverted++; + if (!result.Corrected) return; + + worker.Corrected++; + for (int i = 0; i < peaks; i++) mz.Data[i] = corrected[i]; + } + + /// Copies what MARS's features are computed from out of a pwiz spectrum. + private void Fill(SpectrumRecord record, Spectrum spectrum, BinaryDataArray mz, BinaryDataArray intensity) + { + Scan? scan = spectrum.ScanList.Scans.Count > 0 ? spectrum.ScanList.Scans[0] : null; + + record.Id = spectrum.Id; + record.Index = spectrum.Index; + record.ScanNumber = ScanNumberOf(spectrum.Id, spectrum.Index); + record.MsLevel = 2; + record.InstrumentConfigurationRef = null; + record.FilterString = scan?.CvParam(CVID.MS_filter_string).Value; + + record.RetentionTime = Minutes(scan?.CvParam(CVID.MS_scan_start_time)); + + // MARS holds injection time in seconds; the cvParam is in milliseconds. + double injectionMs = scan?.CvParamValueOrDefault(CVID.MS_ion_injection_time, 0.0) ?? 0.0; + record.InjectionTime = injectionMs > 0 ? injectionMs / 1000.0 : null; + + record.PrecursorMzCenter = 0; + record.PrecursorMzLow = 0; + record.PrecursorMzHigh = 0; + if (spectrum.Precursors.Count > 0) + { + IsolationWindow window = spectrum.Precursors[0].IsolationWindow; + double target = window.CvParamValueOrDefault(CVID.MS_isolation_window_target_m_z, 0.0); + double lower = window.CvParamValueOrDefault(CVID.MS_isolation_window_lower_offset, 0.0); + double upper = window.CvParamValueOrDefault(CVID.MS_isolation_window_upper_offset, 0.0); + record.PrecursorMzCenter = target; + record.PrecursorMzLow = target - lower; + record.PrecursorMzHigh = target + upper; + } + + record.ReportedTic = spectrum.Params.CvParamValueOrDefault(CVID.MS_total_ion_current, 0.0); + + int peaks = mz.Data.Count; + if (record.MzArray.Length < peaks) record.MzArray = new double[peaks]; + if (record.IntensityArray.Length < peaks) record.IntensityArray = new double[peaks]; + + // Summed here rather than taken from the TIC cvParam, because that is what the Python + // matcher did and the log_tic and tic_injection_time features are defined on it. + double summed = 0; + for (int i = 0; i < peaks; i++) + { + record.MzArray[i] = mz.Data[i]; + double value = intensity.Data[i]; + record.IntensityArray[i] = value; + summed += value; + } + + record.PeakCount = peaks; + record.SummedIntensity = summed; + record.AcquisitionStartTime = _acquisitionStart; + record.AbsoluteTime = (_acquisitionStart ?? 0) + (record.RetentionTime * 60.0); + } + + private static long Elapsed(long since) => + (long)((System.Diagnostics.Stopwatch.GetTimestamp() - since) + * (10_000_000.0 / System.Diagnostics.Stopwatch.Frequency)); + + private static void AddElapsed(ref long target, long since) => + Interlocked.Add(ref target, Elapsed(since)); + + /// + /// A time cvParam in minutes, honouring the unit it declares. + /// + /// + /// Vendors differ: Thermo records scan start time in minutes, Bruker in seconds. Reading + /// the value and assuming minutes made a 64-minute diaPASEF run look like 64 hours, which + /// would have gone into the absolute_time feature and out again as noise. An absent or + /// unrecognized unit is treated as minutes, which is mzML's default. + /// + private static double Minutes(CVParam? param) + { + if (param is null) return 0.0; + + double value = param; + return param.Units switch + { + CVID.UO_second => value / 60.0, + CVID.UO_millisecond => value / 60_000.0, + _ => value, + }; + } + + /// + /// Pulls the scan number out of a nativeID, falling back to the index. MARS uses this only + /// for reporting, but a wrong number in a warning sends someone to the wrong spectrum. + /// + private static int ScanNumberOf(string id, int fallback) + { + const string marker = "scan="; + int at = id.LastIndexOf(marker, StringComparison.Ordinal); + if (at < 0) return fallback; + + int start = at + marker.Length; + int end = start; + while (end < id.Length && char.IsDigit(id[end])) end++; + + return end > start && + int.TryParse(id.AsSpan(start, end - start), NumberStyles.Integer, + CultureInfo.InvariantCulture, out int scan) + ? scan + : fallback; + } + + /// + /// One thread's scratch space and running totals. + /// + /// + /// is safe to share - it holds only the model and the + /// options - provided every caller brings its own record, workspace and output buffer. + /// This is what makes the correction parallel without any locking in the hot loop. + /// + private sealed class Worker + { + public readonly SpectrumRecord Record = new(); + + public readonly CorrectionWorkspace Workspace = new(); + + public double[] Corrections = Array.Empty(); + + public long Seen; + + public long Corrected; + + public long MonotonicityFixes; + + public long Reverted; + + public long Ticks; + } +} diff --git a/dotnet/MARS.Pwiz/MzMLEncoding.cs b/dotnet/MARS.Pwiz/MzMLEncoding.cs new file mode 100644 index 0000000..0be65f3 --- /dev/null +++ b/dotnet/MARS.Pwiz/MzMLEncoding.cs @@ -0,0 +1,123 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using System.Text; + +namespace MARS.Pwiz; + +/// How one binary array is encoded. +public readonly struct ArrayEncoding +{ + public ArrayEncoding(bool bits64, bool zlib) + { + Bits64 = bits64; + Zlib = zlib; + } + + public bool Bits64 { get; } + + public bool Zlib { get; } + + /// What msconvert writes unless told otherwise. + public static ArrayEncoding Default => new(bits64: true, zlib: true); + + public override string ToString() => (Bits64 ? "64-bit" : "32-bit") + (Zlib ? " zlib" : " uncompressed"); +} + +/// Encodings for the two arrays MARS cares about. +public readonly struct SpectrumEncoding +{ + public SpectrumEncoding(ArrayEncoding mz, ArrayEncoding intensity) + { + Mz = mz; + Intensity = intensity; + } + + public ArrayEncoding Mz { get; } + + public ArrayEncoding Intensity { get; } + + public static SpectrumEncoding Default => new(ArrayEncoding.Default, ArrayEncoding.Default); + + public override string ToString() => $"m/z {Mz}, intensity {Intensity}"; +} + +/// +/// Reads how an mzML encodes its binary arrays, so a pwiz-backed write can match it. +/// +/// +/// +/// This matters more than it sounds. pwiz's BinaryEncoderConfig defaults to 64-bit +/// uncompressed, and taking that default on a Stellar run produced a file 61% larger +/// than the input. Matching what the input actually used brings it back within 1%. +/// +/// +/// Read per array, not per file, because m/z is commonly 64-bit where intensity is 32-bit and +/// the compression can differ between two arrays of one spectrum. This is still a +/// simplification of what MARS's own writer does: the byte-splice reads the encoding of every +/// array it rewrites, whereas pwiz's config is global with per-array overrides, so a file +/// whose encoding varies from spectrum to spectrum cannot be reproduced exactly. Such a file +/// is unusual - a converter picks an encoding and holds it - and the first spectrum carrying +/// both arrays is taken as representative. +/// +/// +public static class MzMLEncoding +{ + private const string MzArray = "MS:1000514"; + private const string IntensityArray = "MS:1000515"; + private const string Bits32 = "MS:1000521"; + private const string Bits64 = "MS:1000523"; + private const string Zlib = "MS:1000574"; + + /// How much of the file to read looking for the first complete spectrum. + private const int ProbeBytes = 8 * 1024 * 1024; + + /// + /// Sniffs the encoding of the first spectrum that carries both arrays. Falls back to + /// for a file this cannot read. + /// + public static SpectrumEncoding Sniff(string path) + { + string text; + try + { + using FileStream stream = File.OpenRead(path); + int length = (int)Math.Min(ProbeBytes, stream.Length); + var buffer = new byte[length]; + int read = stream.Read(buffer, 0, length); + text = Encoding.UTF8.GetString(buffer, 0, read); + } + catch (IOException) + { + return SpectrumEncoding.Default; + } + + ArrayEncoding? mz = null; + ArrayEncoding? intensity = null; + + int at = text.IndexOf("= 0 && (mz is null || intensity is null)) + { + int end = text.IndexOf("", at, StringComparison.Ordinal); + if (end < 0) break; + + ReadOnlySpan element = text.AsSpan(at, end - at); + var encoding = new ArrayEncoding( + bits64: !Contains(element, Bits32) || Contains(element, Bits64), + zlib: Contains(element, Zlib)); + + if (mz is null && Contains(element, MzArray)) mz = encoding; + else if (intensity is null && Contains(element, IntensityArray)) intensity = encoding; + + at = text.IndexOf(" element, string accession) => + element.IndexOf(accession.AsSpan(), StringComparison.Ordinal) >= 0; +} diff --git a/dotnet/MARS.Pwiz/PwizOutput.cs b/dotnet/MARS.Pwiz/PwizOutput.cs new file mode 100644 index 0000000..e533756 --- /dev/null +++ b/dotnet/MARS.Pwiz/PwizOutput.cs @@ -0,0 +1,233 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using MARS.Core; + +namespace MARS.Pwiz; + +/// An output format MARS can write. +public enum MarsOutputFormat +{ + /// mzML. MARS writes this itself, by splicing bytes into a copy of the input. + MzML, + + /// mzXML 3.2, through pwiz. + MzXml, + + /// mzMLb, mzML in an HDF5 container, through pwiz. + MzMLb, + + /// Mascot Generic Format - MS/MS peak lists only - through pwiz. + Mgf, +} + +/// What a pwiz-backed write did. +public sealed class PwizWriteResult +{ + public long SpectraSeen { get; init; } + + public long SpectraCorrected { get; init; } + + public long MonotonicityFixes { get; init; } + + public long SpectraReverted { get; init; } + + public long OutputLength { get; init; } + + /// Time spent pulling spectra from the input. + public System.TimeSpan ReaderTime { get; init; } + + /// Time spent applying the model. + public System.TimeSpan CorrectorTime { get; init; } +} + +/// +/// Everything one pwiz-backed write needs. Deliberately free of pwiz types so that +/// MARS can reference this assembly whether or not pwiz-sharp was available at build +/// time. +/// +public sealed class PwizWriteRequest +{ + public required string InputPath { get; init; } + + public required string OutputPath { get; init; } + + public required MarsOutputFormat Format { get; init; } + + /// The fitted model, or null to copy spectra through uncorrected. + public MzCalibrator? Calibrator { get; init; } + + public CorrectionOptions Options { get; init; } = new(); + + /// Run start as a Unix timestamp, for the absolute_time feature. + public double? AcquisitionStartTime { get; init; } + + public TemperatureSet? Temperatures { get; init; } + + /// + /// How to encode the binary arrays. Defaults to what msconvert writes; callers should + /// pass of the input so the output matches it. + /// + /// + /// pwiz's own default is 64-bit UNCOMPRESSED, which inflated a Stellar run by 61% before + /// this was set deliberately. An output larger than it needs to be is a cost the user did + /// not ask for. + /// + public SpectrumEncoding Encoding { get; init; } = SpectrumEncoding.Default; + + /// + /// Threads to score the model on. Zero or less means one per processor. + /// + /// + /// Scoring is where a conversion's time goes - 79% of one measured Astral write - and + /// pwiz's writers pull spectra one at a time, so without this it all runs on one core. + /// + public int Threads { get; init; } +} + +/// +/// Writes MARS-corrected spectra in the formats pwiz can serialize. +/// +/// +/// +/// mzML normally does not come through here: when the input is itself an mzML, MARS writes it +/// by splicing corrected bytes into a copy, which keeps every byte it did not change identical +/// by construction (docs/mzml-passthrough.md). Splicing needs an input to copy, so mzML +/// written from a vendor file does come through here - there is nothing to splice into, and +/// the file has to be built. Deciding between the two is the caller's job; see +/// CorrectedFileWriter. +/// +/// +/// When MARS is built without a pwiz-sharp checkout, is false and +/// throws. Everything else about MARS is unaffected. +/// +/// +public static partial class PwizOutput +{ + /// Whether this build can write the pwiz-backed formats. + public static bool Available => +#if MARS_NO_PWIZ + false; +#else + true; +#endif + + /// + /// The formats this build can write here, mzML included. + /// + /// + /// mzMLb is dropped on anything but x64: it is HDF5, and the native library that writes it + /// is published for x64 alone. Listing it on arm64 would be a promise broken at the moment + /// someone asked for it. + /// + public static IReadOnlyList Supported + { + get + { + if (!Available) return new[] { MarsOutputFormat.MzML }; + + var formats = new List + { + MarsOutputFormat.MzML, + MarsOutputFormat.MzXml, + MarsOutputFormat.Mgf, + }; + + if (System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture + == System.Runtime.InteropServices.Architecture.X64) + { + formats.Insert(2, MarsOutputFormat.MzMLb); + } + + return formats; + } + } + + /// Parses a format name, case-insensitively. Returns false for anything else. + public static bool TryParse(string? name, out MarsOutputFormat format) + { + switch (name?.Trim().ToLowerInvariant()) + { + case null or "" or "mzml": format = MarsOutputFormat.MzML; return true; + case "mzxml": format = MarsOutputFormat.MzXml; return true; + case "mzmlb": format = MarsOutputFormat.MzMLb; return true; + case "mgf": format = MarsOutputFormat.Mgf; return true; + default: format = MarsOutputFormat.MzML; return false; + } + } + + /// The file extension for a format, including the dot. + public static string Extension(MarsOutputFormat format) => format switch + { + MarsOutputFormat.MzML => ".mzML", + MarsOutputFormat.MzXml => ".mzXML", + MarsOutputFormat.MzMLb => ".mzMLb", + MarsOutputFormat.Mgf => ".mgf", + _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unknown output format."), + }; + + /// The name a user types for a format. + public static string Name(MarsOutputFormat format) => format switch + { + MarsOutputFormat.MzML => "mzML", + MarsOutputFormat.MzXml => "mzXML", + MarsOutputFormat.MzMLb => "mzMLb", + MarsOutputFormat.Mgf => "mgf", + _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unknown output format."), + }; + + /// + /// Whether a format loses information relative to mzML, so callers can warn rather than + /// let a user discover it downstream. + /// + public static string? LossWarning(MarsOutputFormat format) => format switch + { + // MGF carries MS2 peak lists and little else: no MS1, no chromatograms, and none of + // the scan metadata MARS's own features are computed from. A corrected MGF cannot be + // fed back to MARS. + MarsOutputFormat.Mgf => + "mgf keeps MS2 peak lists only - no MS1 spectra, no chromatograms, and none of the " + + "scan metadata MARS reads. The result cannot be re-calibrated or re-analysed by " + + "MARS.", + + // mzXML predates most of the CV vocabulary and cannot express ion mobility or several + // isolation-window terms. + MarsOutputFormat.MzXml => + "mzXML cannot express everything mzML can - ion mobility and some isolation-window " + + "terms have nowhere to go. Prefer mzML or mzMLb unless a downstream tool requires " + + "mzXML.", + + _ => null, + }; + + /// + /// Reads 's input, applies the model, and writes the result in + /// the requested format. + /// + /// + /// This build has no pwiz-sharp, or the format is one pwiz cannot write. + /// + public static PwizWriteResult Write(PwizWriteRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + RequireAvailable(request.Format); + +#if MARS_NO_PWIZ + throw new NotSupportedException("unreachable: RequireAvailable throws first."); +#else + return PwizWriteBackend.Write(request); +#endif + } + + private static void RequireAvailable(MarsOutputFormat format) + { + if (Available) return; + + throw new NotSupportedException( + $"This build of MARS cannot write {Name(format)}: it was built without a pwiz-sharp " + + "checkout. Rebuild with -p:PwizSharpDir=/pwiz/pwiz-sharp, or write mzML, " + + "which MARS writes itself."); + } +} diff --git a/dotnet/MARS.Pwiz/PwizSpectrumSource.cs b/dotnet/MARS.Pwiz/PwizSpectrumSource.cs new file mode 100644 index 0000000..4edd235 --- /dev/null +++ b/dotnet/MARS.Pwiz/PwizSpectrumSource.cs @@ -0,0 +1,316 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using MARS.IO; +using MARS.Core; +using Pwiz.Data.Common; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.Common.Params; +using Pwiz.Data.MsData; +using Pwiz.Data.MsData.Instruments; +using Pwiz.Data.MsData.Readers; +using Pwiz.Data.MsData.Spectra; + +namespace MARS.Pwiz; + +/// +/// Reads spectra through pwiz, so MARS can open a Thermo .raw without converting it +/// first. +/// +/// +/// +/// Removing the conversion is the point. It is not faster to read - a 4.9 GB Astral run takes +/// about 72 s through the vendor SDK against 41 s for the equivalent mzML, and it does not +/// thread - but it removes an msconvert pass and the ~5 GB intermediate it leaves behind. +/// +/// +/// One is reused across the enumeration, matching what +/// MzMLFile.ReadSpectra does, so a consumer that keeps the arrays must copy them. +/// +/// +internal sealed class PwizSpectrumSource : ISpectrumSource +{ + private readonly MSData _msd = new(); + private readonly ISpectrumList _spectra; + private readonly IVendorCentroidingSpectrumList? _centroider; + + /// + /// Registers the vendor readers before any instance exists, so that constructing one is + /// enough - no caller has to remember. The runtime runs this before the first instance + /// constructor. + /// + static PwizSpectrumSource() => VendorReaders.EnsureRegistered(); + + public PwizSpectrumSource(string path) + { + Path = path; + Length = LengthOf(path); + + // Collapse the ion mobility dimension. pwiz otherwise presents an uncombined TIMS + // frame as hundreds of spectra that share one retention time and one isolation m/z, + // separated only by mobility - ProteoWizard's own diaPASEF.d is 4,631 spectra at five + // distinct scan times. MARS has no mobility feature and does not want one: combining + // sums each frame's mobility scans back into one spectrum per isolation window, which + // is the shape every other instrument already produces and the shape the matcher and + // the space-charge features assume. + ReaderList.Default.Read(path, _msd, new ReaderConfig { CombineIonMobilitySpectra = true }); + _spectra = _msd.Run.SpectrumList + ?? throw new InvalidDataException($"No spectra in {path}."); + + _centroider = _spectra as IVendorCentroidingSpectrumList; + + AcquisitionStartTime = StartTimeOf(_msd); + Analyzer = DetectAnalyzer(_msd, _spectra); + } + + public string Path { get; } + + public long Length { get; } + + public double? AcquisitionStartTime { get; } + + public MassAnalyzerClass Analyzer { get; } + + public IEnumerable ReadSpectra(int? msLevel = 2) + { + var record = new SpectrumRecord(); + + for (int i = 0; i < _spectra.Count; i++) + { + // Metadata first: deciding whether this spectrum is wanted before decoding its + // arrays is most of the saving when only MS2 is being read. + Spectrum probe = _spectra.GetSpectrum(i, DetailLevel.FullMetadata); + int level = probe.Params.CvParamValueOrDefault(CVID.MS_ms_level, 0); + if (msLevel.HasValue && level != msLevel.Value) continue; + + Spectrum spectrum = Read(i); + if (Fill(record, spectrum, level)) yield return record; + } + } + + /// + /// Reads one spectrum, centroided by the vendor when the run is stored as profile. + /// + /// + /// + /// Profile data is a sampled curve, not a peak list. A Sciex ZenoTOF writes it that way: + /// 1,619 points in one MS2, evenly spaced at 0.00233 Th, which is 16 ppm at m/z 142. MARS + /// measures mass error by taking the most intense peak inside a tolerance window, so on + /// profile data the answer is quantised to the sampling grid - a floor several times larger + /// than the error an instrument like this actually has. The space-charge features fare + /// worse: they count peaks around a match, and on profile they would count samples of the + /// same ion. + /// + /// + /// The vendor's own centroiding is used rather than a peak picker of ours, because the + /// vendor knows its detector. pwiz exposes it through + /// , which the Sciex list implements. + /// + /// + private Spectrum Read(int index) + { + if (_centroider is null) return _spectra.GetSpectrum(index, getBinaryData: true); + + Spectrum spectrum = _spectra.GetSpectrum(index, getBinaryData: true); + return spectrum.Params.HasCVParam(CVID.MS_profile_spectrum) + ? _centroider.GetCentroidSpectrum(index, getBinaryData: true) + : spectrum; + } + + public void Dispose() + { + _spectra.Dispose(); + _msd.Dispose(); + } + + /// Copies one pwiz spectrum into the record MARS's matcher consumes. + private bool Fill(SpectrumRecord record, Spectrum spectrum, int msLevel) + { + BinaryDataArray? mz = spectrum.GetMZArray(); + BinaryDataArray? intensity = spectrum.GetIntensityArray(); + if (mz is null || intensity is null) return false; + + Scan? scan = spectrum.ScanList.Scans.Count > 0 ? spectrum.ScanList.Scans[0] : null; + + record.Id = spectrum.Id; + record.Index = spectrum.Index; + record.ScanNumber = ScanNumberOf(spectrum.Id, spectrum.Index); + record.MsLevel = msLevel; + record.InstrumentConfigurationRef = null; + record.FilterString = scan?.CvParam(CVID.MS_filter_string).Value; + + record.RetentionTime = Minutes(scan?.CvParam(CVID.MS_scan_start_time)); + + // MARS holds injection time in seconds; the cvParam is milliseconds. + double injectionMs = scan?.CvParamValueOrDefault(CVID.MS_ion_injection_time, 0.0) ?? 0.0; + record.InjectionTime = injectionMs > 0 ? injectionMs / 1000.0 : null; + + record.PrecursorMzCenter = 0; + record.PrecursorMzLow = 0; + record.PrecursorMzHigh = 0; + if (spectrum.Precursors.Count > 0) + { + IsolationWindow window = spectrum.Precursors[0].IsolationWindow; + double target = window.CvParamValueOrDefault(CVID.MS_isolation_window_target_m_z, 0.0); + double lower = window.CvParamValueOrDefault(CVID.MS_isolation_window_lower_offset, 0.0); + double upper = window.CvParamValueOrDefault(CVID.MS_isolation_window_upper_offset, 0.0); + record.PrecursorMzCenter = target; + record.PrecursorMzLow = target - lower; + record.PrecursorMzHigh = target + upper; + } + + record.ReportedTic = spectrum.Params.CvParamValueOrDefault(CVID.MS_total_ion_current, 0.0); + + int peaks = mz.Data.Count; + if (record.MzArray.Length < peaks) record.MzArray = new double[peaks]; + if (record.IntensityArray.Length < peaks) record.IntensityArray = new double[peaks]; + + // Summed rather than taken from the TIC cvParam, because that is what the Python + // matcher did and log_tic and tic_injection_time are defined on it. + double summed = 0; + for (int i = 0; i < peaks; i++) + { + record.MzArray[i] = mz.Data[i]; + double value = intensity.Data[i]; + record.IntensityArray[i] = value; + summed += value; + } + + record.PeakCount = peaks; + record.SummedIntensity = summed; + record.AcquisitionStartTime = AcquisitionStartTime; + record.AbsoluteTime = (AcquisitionStartTime ?? 0) + (record.RetentionTime * 60.0); + return true; + } + + /// + /// A time cvParam in minutes, honouring the unit it declares. + /// + /// + /// Vendors differ: Thermo records scan start time in minutes, Bruker in seconds. Reading + /// the value and assuming minutes made a 64-minute diaPASEF run look like 64 hours, which + /// would have gone into the absolute_time feature and out again as noise. An absent or + /// unrecognized unit is treated as minutes, which is mzML's default. + /// + private static double Minutes(CVParam? param) + { + if (param is null) return 0.0; + + double value = param; + return param.Units switch + { + CVID.UO_second => value / 60.0, + CVID.UO_millisecond => value / 60_000.0, + _ => value, + }; + } + + /// + /// Works out which analyzer recorded the MS2 spectra. + /// + /// + /// Read from the first MS2 spectrum's own configuration rather than the run default, + /// because on a hybrid instrument those differ: an Orbitrap Astral file names the + /// orbitrap as the run default, since that takes the MS1 survey, and points only its MS2 + /// spectra at the Astral analyzer. MS2 is what MARS calibrates, so that is what decides. + /// + private static MassAnalyzerClass DetectAnalyzer(MSData msd, ISpectrumList spectra) + { + for (int i = 0; i < spectra.Count; i++) + { + Spectrum spectrum = spectra.GetSpectrum(i, DetailLevel.FullMetadata); + if (spectrum.Params.CvParamValueOrDefault(CVID.MS_ms_level, 0) != 2) continue; + + Scan? scan = spectrum.ScanList.Scans.Count > 0 ? spectrum.ScanList.Scans[0] : null; + InstrumentConfiguration? configuration = + scan?.InstrumentConfiguration ?? msd.Run.DefaultInstrumentConfiguration; + + MassAnalyzerClass analyzer = Classify(configuration); + if (analyzer != MassAnalyzerClass.Unknown) return analyzer; + + // The configuration did not settle it. Thermo's filter string does: ITMS, FTMS + // and ASTMS name the analyzer at the front of every filter. + return MassAnalyzers.ClassifyFilterString(scan?.CvParam(CVID.MS_filter_string).Value); + } + + // No MS2 at all. Fall back to the run default so that a QC pass over an MS1-only file + // still reports on the right scale. + return Classify(msd.Run.DefaultInstrumentConfiguration); + } + + /// + /// Classifies one instrument configuration by its measuring analyzer - the highest-order + /// component that is not the isolating quadrupole. + /// + private static MassAnalyzerClass Classify(InstrumentConfiguration? configuration) + { + if (configuration is null) return MassAnalyzerClass.Unknown; + + var analyzers = new List<(int Order, string Accession)>(); + foreach (Component component in configuration.ComponentList) + { + if (component.Type != ComponentType.Analyzer) continue; + foreach (CVParam param in component.CVParams) + { + // pwiz identifies terms by enum; MARS matches on the accession string, which + // is what the mzML actually carries. + string accession = CvLookup.CvTermInfo(param.Cvid).Id; + analyzers.Add((component.Order, accession)); + } + } + + return MassAnalyzers.Classify(MassAnalyzers.MeasuringAnalyzer(analyzers)); + } + + /// Run start as a Unix timestamp, for the absolute_time feature. + /// + /// Parsed by the mzML reader's own routine rather than a second copy of the rules here. A + /// stamp with no UTC offset has to be read the same way in both, or the same run gives one + /// absolute_time through its .raw and another through the mzML msconvert made from it - + /// shifted by the machine's offset from UTC, and only on machines that have one. + /// + private static double? StartTimeOf(MSData msd) + { + string? stamp = msd.Run.StartTimeStamp; + return string.IsNullOrEmpty(stamp) ? null : MzMLSpectrumParser.ParseStartTimeStamp(stamp); + } + + /// A Thermo .raw is a file; other vendors use a directory. + private static long LengthOf(string path) + { + try + { + if (File.Exists(path)) return new FileInfo(path).Length; + if (!Directory.Exists(path)) return 0; + + long total = 0; + foreach (string file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) + total += new FileInfo(file).Length; + return total; + } + catch (IOException) + { + return 0; + } + } + + private static int ScanNumberOf(string id, int fallback) + { + const string marker = "scan="; + int at = id.LastIndexOf(marker, StringComparison.Ordinal); + if (at < 0) return fallback; + + int start = at + marker.Length; + int end = start; + while (end < id.Length && char.IsDigit(id[end])) end++; + + return end > start && + int.TryParse(id.AsSpan(start, end - start), NumberStyles.Integer, + CultureInfo.InvariantCulture, out int scan) + ? scan + : fallback; + } +} diff --git a/dotnet/MARS.Pwiz/PwizWriteBackend.cs b/dotnet/MARS.Pwiz/PwizWriteBackend.cs new file mode 100644 index 0000000..9953004 --- /dev/null +++ b/dotnet/MARS.Pwiz/PwizWriteBackend.cs @@ -0,0 +1,103 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.MsData; +using Pwiz.Data.MsData.Encoding; +using Pwiz.Data.MsData.Readers; + +namespace MARS.Pwiz; + +/// +/// The pwiz side of a write: read, wrap with MARS's correction, serialize. +/// +/// +/// Compiled only when a pwiz-sharp checkout was available. Everything that has to exist +/// unconditionally lives in . +/// +internal static class PwizWriteBackend +{ + /// Same reason as PwizSpectrumSource's: registration should not be a thing + /// a caller can forget. + static PwizWriteBackend() => VendorReaders.EnsureRegistered(); + + public static PwizWriteResult Write(PwizWriteRequest request) + { + string? directory = Path.GetDirectoryName(Path.GetFullPath(request.OutputPath)); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + + using var msd = new MSData(); + // Combined the same way the reader does, so that what is written matches what was + // matched and modelled. Reading a frame one way and writing it another would put + // corrections on spectra the model never saw. + ReaderList.Default.Read( + request.InputPath, msd, new ReaderConfig { CombineIonMobilitySpectra = true }); + + if (msd.Run.SpectrumList is null) + throw new InvalidDataException($"No spectra in {request.InputPath}."); + + var mars = new MarsSpectrumList( + msd.Run.SpectrumList, + request.Calibrator, + request.Options, + request.AcquisitionStartTime, + request.Temperatures, + request.Threads); + + msd.Run.SpectrumList = mars; + + MSDataFile.Write(msd, request.OutputPath, new WriteConfig + { + Format = FormatOf(request.Format), + Indexed = true, + EncoderConfig = EncoderFor(request.Encoding), + }); + + return new PwizWriteResult + { + SpectraSeen = mars.SpectraSeen, + SpectraCorrected = mars.SpectraCorrected, + MonotonicityFixes = mars.MonotonicityFixes, + SpectraReverted = mars.SpectraReverted, + OutputLength = new FileInfo(request.OutputPath).Length, + ReaderTime = mars.ReaderTime, + CorrectorTime = mars.CorrectorTime, + }; + } + + private static WriteFormat FormatOf(MarsOutputFormat format) => format switch + { + // Reached only when the input was a vendor file: an mzML input is spliced instead. + MarsOutputFormat.MzML => WriteFormat.Mzml, + MarsOutputFormat.MzXml => WriteFormat.MzXml, + MarsOutputFormat.MzMLb => WriteFormat.MzMLb, + MarsOutputFormat.Mgf => WriteFormat.Mgf, + _ => throw new ArgumentOutOfRangeException( + nameof(format), format, "No pwiz writer for this format."), + }; + + /// + /// Builds an encoder that matches the input rather than taking pwiz's defaults. + /// + /// + /// The default is 64-bit uncompressed, which made a Stellar run 61% larger than its input + /// before this was set. The base config carries the m/z encoding and the intensity array + /// gets a per-array override, since the two commonly differ. + /// + private static BinaryEncoderConfig EncoderFor(SpectrumEncoding encoding) + { + var config = new BinaryEncoderConfig + { + Precision = encoding.Mz.Bits64 ? BinaryPrecision.Bits64 : BinaryPrecision.Bits32, + Compression = encoding.Mz.Zlib ? BinaryCompression.Zlib : BinaryCompression.None, + }; + + config.PrecisionOverrides[CVID.MS_intensity_array] = + encoding.Intensity.Bits64 ? BinaryPrecision.Bits64 : BinaryPrecision.Bits32; + config.CompressionOverrides[CVID.MS_intensity_array] = + encoding.Intensity.Zlib ? BinaryCompression.Zlib : BinaryCompression.None; + + return config; + } +} diff --git a/dotnet/MARS.Pwiz/SpectrumSources.cs b/dotnet/MARS.Pwiz/SpectrumSources.cs new file mode 100644 index 0000000..33f56d8 --- /dev/null +++ b/dotnet/MARS.Pwiz/SpectrumSources.cs @@ -0,0 +1,162 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using MARS.Core; +using MARS.IO; + +namespace MARS.Pwiz; + +/// +/// Opens a run, choosing a reader from the file rather than from a flag. +/// +/// +/// mzML goes to MARS's own reader, which is faster and is what the byte-splice writer needs +/// the byte offsets from. Everything else goes to pwiz. A user should not have to tell MARS +/// which of its readers to use when the extension already says. +/// +public static class SpectrumSources +{ + /// Extensions MARS can read without pwiz. + private static readonly string[] NativeExtensions = { ".mzml" }; + + /// + /// Vendor formats pwiz can read. Thermo, Bruker and Sciex are referenced; the rest are + /// listed so that MARS can say "this build cannot read that" rather than "unrecognized + /// file", and so that a format gated by platform - Sciex off Windows - gives the same + /// answer. + /// + private static readonly Dictionary VendorExtensions = new(StringComparer.OrdinalIgnoreCase) + { + [".raw"] = "Thermo", + [".wiff"] = "Sciex", + [".wiff2"] = "Sciex", + + // Bruker and Agilent both use a .d directory; pwiz's readers identify which by what is + // inside it, so MARS does not have to guess here. + [".d"] = "Bruker or Agilent", + [".tdf"] = "Bruker", + [".tsf"] = "Bruker", + [".baf"] = "Bruker", + [".lcd"] = "Shimadzu", + [".uimf"] = "UIMF", + }; + + /// + /// Vendors this build carries a reader for AND can run on this machine. + /// + /// + /// Architecture matters, not only the build. Thermo's SDK is managed and runs anywhere; + /// Bruker's and Sciex's are native x64, so an arm64 build stages libraries it cannot load. + /// Advertising them there would be a promise broken at the moment someone opens a file, + /// which is exactly what this list exists to avoid. + /// + private static readonly HashSet LinkedVendors = BuildLinkedVendors(); + + private static HashSet BuildLinkedVendors() + { + var vendors = new HashSet(StringComparer.Ordinal); +#if !MARS_NO_PWIZ + vendors.Add("Thermo"); + + bool nativeX64 = RuntimeInformation.ProcessArchitecture == Architecture.X64; + if (nativeX64) + { + vendors.Add("Bruker or Agilent"); + vendors.Add("Bruker"); +#if MARS_SCIEX + vendors.Add("Sciex"); +#endif + } +#endif + return vendors; + } + + /// True when this build can actually open the path. + /// + /// Used for directory scanning, where picking up a file that cannot be opened would turn + /// a usable folder into a failed run. A path named explicitly goes through + /// instead, so that it gets a reason rather than silence. + /// + public static bool IsReadable(string path) => + IsNative(path) || + (VendorExtensions.TryGetValue(Path.GetExtension(path), out string? vendor) && + LinkedVendors.Contains(vendor)); + + /// + /// True when MARS knows what the format is, whether or not this build can open it. A + /// recognized-but-unavailable format earns an explanation from . + /// + public static bool IsRecognized(string path) => + IsNative(path) || VendorExtensions.ContainsKey(Path.GetExtension(path)); + + /// True when this path is an mzML, which MARS reads itself. + public static bool IsNative(string path) => + Array.IndexOf(NativeExtensions, Path.GetExtension(path).ToLowerInvariant()) >= 0; + + /// + /// Whether writing mzML for this input can use the byte-splice writer. + /// + /// + /// Only an mzML input can be spliced, because splicing means copying the input and + /// replacing the ranges that changed. A vendor file has no mzML to copy, so its mzML has + /// to be built - the guarantee does not apply and cannot be pretended at. + /// + public static bool CanSplice(string path) => IsNative(path); + + /// Opens a run for reading. + /// + /// The format needs pwiz and this build has none, or MARS does not recognize it. + /// + public static ISpectrumSource Open(string path) + { + if (IsNative(path)) return new MzMLSpectrumSource(path); + + string extension = Path.GetExtension(path); + if (!VendorExtensions.TryGetValue(extension, out string? vendor)) + { + throw new NotSupportedException( + $"MARS does not recognize '{extension}'. Expected .mzML or a vendor format " + + "MARS was built to read."); + } + +#if MARS_NO_PWIZ + throw new NotSupportedException( + $"Reading {vendor} data ({extension}) needs pwiz-sharp, and this build of MARS was " + + "made without it. Rebuild with -p:PwizSharpDir=/pwiz/pwiz-sharp " + + "-p:IAgreeToVendorLicenses=true, or convert to mzML first."); +#else + try + { + return new PwizSpectrumSource(path); + } + catch (Exception ex) when (ex is TypeInitializationException or DllNotFoundException + or FileNotFoundException or BadImageFormatException) + { + // The vendor SDK is gated behind IAgreeToVendorLicenses at pwiz build time, so a + // build that has pwiz but not the SDK fails here rather than at compile time. + // Saying which knob is missing beats a load-failure stack trace. + throw new NotSupportedException( + $"This build cannot read {vendor} data ({extension}): the vendor SDK was not " + + "available when pwiz-sharp was built. Rebuild pwiz-sharp with " + + $"-p:IAgreeToVendorLicenses=true. ({ex.GetType().Name}: {ex.Message})", ex); + } +#endif + } + + /// + /// What this build can read, for --version. Only formats it actually carries a + /// reader for - a list that promised Shimadzu because the name appears in a table would be + /// worse than no list. + /// + public static IEnumerable ReadableExtensions() + { + yield return ".mzML"; + foreach (KeyValuePair entry in VendorExtensions) + { + if (LinkedVendors.Contains(entry.Value)) yield return entry.Key; + } + } +} diff --git a/dotnet/MARS.Pwiz/VendorReaders.cs b/dotnet/MARS.Pwiz/VendorReaders.cs new file mode 100644 index 0000000..42c8de6 --- /dev/null +++ b/dotnet/MARS.Pwiz/VendorReaders.cs @@ -0,0 +1,68 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using Pwiz.Data.MsData.Readers; +using Pwiz.Vendor.Bruker; +using Pwiz.Vendor.Thermo; +#if MARS_SCIEX +using Pwiz.Vendor.Sciex; +#endif + +namespace MARS.Pwiz; + +/// +/// Plugs the vendor readers MARS was built with into pwiz's format dispatcher. +/// +/// +/// +/// Pwiz.Data.MsData deliberately does not reference the vendor projects - that would +/// drag every encrypted vendor SDK into everything that touches the core data model - so +/// ReaderList.Default knows only the open formats until a host adds the rest. Without +/// this, opening a .raw fails with "no registered reader recognized the file" even +/// though the reader is sitting in the same output directory. +/// +/// +/// Registered from a static constructor rather than from Main, because MARS reaches +/// pwiz from several places - reading a run, writing one - and a registration that depends on +/// the entry point having remembered to call it is one that will eventually be missed. Both +/// entry points inside this assembly call , and the runtime +/// guarantees the static constructor runs exactly once however many of them do. +/// +/// +/// A module initializer would be tidier still and was the first attempt, but CA2255 objects to +/// one in a library and is right to: it would run on assembly load, which is a side effect a +/// caller has no way to anticipate. A static constructor runs on first use instead. +/// +/// +internal static class VendorReaders +{ + static VendorReaders() => Register(); + + /// + /// Ensures the vendor readers are registered. Cheap and idempotent: the work happens in + /// the static constructor, which the runtime runs once on first touch of this type. + /// + internal static void EnsureRegistered() + { + // Referencing the type is what triggers the static constructor; there is nothing to do + // in the body. + } + + private static void Register() + { + // Appended once. ReaderList.Default rebuilds a list on every access and copies + // AdditionalReaders into it, so registering twice would double every vendor reader. + if (ReaderList.AdditionalReaders.Count > 0) return; + + // AdditionalReaders is a List; ThermoReaderRegistration.AddTo wants a + // ReaderList, and ReaderList.Default builds a fresh list each time it is read, so + // adding to that would be adding to a copy that is thrown away. + ReaderList.AdditionalReaders.Add(new Reader_Thermo()); + ReaderList.AdditionalReaders.Add(new Reader_Bruker()); + + // Sciex only where its SDK runs. Everywhere else SpectrumSources still recognizes + // .wiff and .wiff2 well enough to say why it cannot open them. +#if MARS_SCIEX + ReaderList.AdditionalReaders.Add(new Reader_Sciex()); +#endif + } +} diff --git a/dotnet/MARS.Test/CalibrationTest.cs b/dotnet/MARS.Test/CalibrationTest.cs new file mode 100644 index 0000000..59daf1a --- /dev/null +++ b/dotnet/MARS.Test/CalibrationTest.cs @@ -0,0 +1,350 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using MARS.Core; +using Xunit; + +namespace MARS.Test; + +public sealed class CalibrationTest : IDisposable +{ + private readonly string _directory; + + public CalibrationTest() + { + _directory = Path.Combine(Path.GetTempPath(), "mars-cal-" + Guid.NewGuid().ToString("N")[..12]); + Directory.CreateDirectory(_directory); + } + + public void Dispose() + { + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + } + } + + /// + /// A synthetic run whose mass error is a known function of two features. The model has + /// to recover it well enough to cut the spread substantially, or nothing downstream + /// means anything. + /// + private static MatchTable BuildSyntheticMatches(int rows = 20000, double noise = 0.01) + { + MarsFeature[] collect = + { + MarsFeature.PrecursorMz, MarsFeature.FragmentMz, MarsFeature.LogTic, + MarsFeature.LogIntensity, MarsFeature.AbsoluteTime, + }; + + var table = new MatchTable(collect); + var random = new Random(20260819); + + for (var i = 0; i < rows; i++) + { + double fragmentMz = 200.0 + (random.NextDouble() * 1000.0); + double absoluteTime = random.NextDouble() * 3600.0; + double intensity = 500.0 + (random.NextDouble() * 100000.0); + + // The truth: error grows with m/z and drifts through the run. + double truth = (fragmentMz * 2.0e-5) + (absoluteTime * 5.0e-6) - 0.01; + double error = truth + ((random.NextDouble() - 0.5) * noise); + + table.Set(MarsFeature.PrecursorMz, 400.0 + (i % 20)); + table.Set(MarsFeature.FragmentMz, fragmentMz); + table.Set(MarsFeature.LogTic, Math.Log10(1.0e6)); + table.Set(MarsFeature.LogIntensity, Math.Log10(intensity)); + table.Set(MarsFeature.AbsoluteTime, absoluteTime); + table.DeltaMz.Add(error); + table.ObservedIntensity.Add(intensity); + // Several rows per peptide, as real matching produces: one peptide is matched in + // many spectra. Cross-validation keeps them together. + table.PeptideGroup.Add(i / 8); + table.CommitRow(); + } + + return table; + } + + [Fact] + public void ModelRecoversAKnownSystematicError() + { + MatchTable table = BuildSyntheticMatches(); + var options = new CalibrationOptions { ImportanceSampleRows = 2000 }; + + MzCalibrator calibrator = MzCalibrator.Fit(table, options, absoluteTimeOffset: 0); + TrainingStatistics stats = calibrator.Statistics!; + + Assert.Equal(20000, stats.RowsUsed); + + // The "after" figures are out-of-fold: each row scored by a model that never trained + // on its peptide. That is a little worse than the in-sample number this threshold + // used to be set against, and deliberately so - it is what the model achieves on data + // it has not seen. A collapse to under 40% of the original spread still means the + // systematic part was found; what is left is the injected noise. + Assert.True(stats.After.StdDev < 0.40 * stats.Before.StdDev, + $"spread should collapse: {stats.Before.StdDev:R} -> {stats.After.StdDev:R}"); + Assert.True(Math.Abs(stats.After.Median) < 0.002, + $"residuals should centre near zero, got {stats.After.Median:R}"); + + // fragment_mz and absolute_time carry the signal; the rest is noise. + double[] importance = stats.PermutationImportance; + int fragmentSlot = calibrator.Features.SlotOf(MarsFeature.FragmentMz); + int timeSlot = calibrator.Features.SlotOf(MarsFeature.AbsoluteTime); + int precursorSlot = calibrator.Features.SlotOf(MarsFeature.PrecursorMz); + Assert.True(importance[fragmentSlot] > importance[precursorSlot]); + Assert.True(importance[timeSlot] > importance[precursorSlot]); + } + + [Fact] + public void TrainingIsReproducible() + { + var options = new CalibrationOptions { ImportanceSampleRows = 0 }; + + MzCalibrator first = MzCalibrator.Fit(BuildSyntheticMatches(4000), options, 0); + MzCalibrator second = MzCalibrator.Fit(BuildSyntheticMatches(4000), options, 0); + + var row = new double[first.Features.Count]; + for (var i = 0; i < row.Length; i++) row[i] = 300.0 + i; + + Assert.Equal(first.PredictDelta(row), second.PredictDelta(row)); + } + + /// + /// Histogram threads must not move a single prediction. This is the invariant that lets + /// MARS use every core without giving up reproducible output. + /// + [Fact] + public void TrainingIsDeterministicAcrossThreadCounts() + { + MzCalibrator single = MzCalibrator.Fit( + BuildSyntheticMatches(6000), + new CalibrationOptions { MaxDegreeOfParallelism = 1, ImportanceSampleRows = 0 }, 0); + + MzCalibrator many = MzCalibrator.Fit( + BuildSyntheticMatches(6000), + new CalibrationOptions { MaxDegreeOfParallelism = 16, ImportanceSampleRows = 0 }, 0); + + var random = new Random(7); + for (var trial = 0; trial < 50; trial++) + { + var row = new double[single.Features.Count]; + for (var i = 0; i < row.Length; i++) row[i] = random.NextDouble() * 1200.0; + Assert.Equal(single.PredictDelta(row), many.PredictDelta(row)); + } + } + + [Fact] + public void ModelFileRoundTripsExactly() + { + MzCalibrator original = MzCalibrator.Fit( + BuildSyntheticMatches(3000), new CalibrationOptions { ImportanceSampleRows = 500 }, 1.7e9); + + string path = Path.Combine(_directory, "model.json"); + MarsModelIo.Save(original, path); + MzCalibrator reloaded = MarsModelIo.Load(path); + + Assert.Equal(original.Features.Names(), reloaded.Features.Names()); + Assert.Equal(original.AbsoluteTimeOffset, reloaded.AbsoluteTimeOffset); + Assert.Equal(original.Options.NEstimators, reloaded.Options.NEstimators); + + var random = new Random(11); + for (var trial = 0; trial < 100; trial++) + { + var row = new double[original.Features.Count]; + for (var i = 0; i < row.Length; i++) row[i] = random.NextDouble() * 2000.0; + Assert.Equal(original.PredictDelta(row), reloaded.PredictDelta(row)); + } + } + + /// + /// The acquisition-time offset has to survive into the model file. Without it, a + /// correction run feeds raw Unix timestamps to a model trained on times re-based to the + /// earliest run, and every inference row lands past the largest value the model saw. + /// + [Fact] + public void AbsoluteTimeOffsetTravelsWithTheModel() + { + const double offset = 1733198754.0; + MzCalibrator calibrator = MzCalibrator.Fit( + BuildSyntheticMatches(2000), new CalibrationOptions { ImportanceSampleRows = 0 }, offset); + + string path = Path.Combine(_directory, "offset.json"); + MarsModelIo.Save(calibrator, path); + + Assert.Equal(offset, MarsModelIo.Load(path).AbsoluteTimeOffset); + } + + [Fact] + public void EmptyTableIsRejected() + { + var empty = new MatchTable(new[] { MarsFeature.FragmentMz }); + Assert.Throws(() => MzCalibrator.Fit(empty, new CalibrationOptions(), 0)); + } + + /// + /// Missing values must never reach the model. Osprey.ML maps NaN to bin 0, while + /// XGBoost learns a per-node default direction, so a NaN slipping through would diverge + /// from the reference in a way that is very hard to trace. + /// + [Fact] + public void RowsWithMissingFeaturesAreDropped() + { + MarsFeature[] collect = { MarsFeature.FragmentMz, MarsFeature.LogIntensity, MarsFeature.InjectionTime }; + var table = new MatchTable(collect); + + for (var i = 0; i < 4000; i++) + { + table.Set(MarsFeature.FragmentMz, 300.0 + (i % 700)); + table.Set(MarsFeature.LogIntensity, 3.0 + ((i % 30) * 0.05)); + + // One row in ten has no injection time, as happens when an instrument omits it. + // The rest vary: a constant injection time is not selected as a feature at all, + // and a row is only dropped for a value missing from a feature in use. + table.Set(MarsFeature.InjectionTime, i % 10 == 0 ? double.NaN : 0.02 + ((i % 13) * 0.001)); + table.DeltaMz.Add(0.001 * (i % 7)); + table.ObservedIntensity.Add(1000.0); + table.PeptideGroup.Add(i / 8); + table.CommitRow(); + } + + MzCalibrator calibrator = MzCalibrator.Fit( + table, new CalibrationOptions { ImportanceSampleRows = 0 }, 0); + + Assert.Equal(4000, calibrator.Statistics!.RowsMatched); + Assert.Equal(3600, calibrator.Statistics.RowsUsed); + } +} + +public sealed class SpectrumCorrectorTest +{ + private static MzCalibrator FitTrivialModel(out MarsFeature[] features) + { + features = new[] { MarsFeature.FragmentMz }; + var table = new MatchTable(features); + + // A constant offset the model can reproduce anywhere. + for (var i = 0; i < 3000; i++) + { + table.Set(MarsFeature.FragmentMz, 200.0 + (i * 0.3)); + table.DeltaMz.Add(0.02); + table.ObservedIntensity.Add(1000.0); + table.PeptideGroup.Add(i / 8); + table.CommitRow(); + } + + return MzCalibrator.Fit(table, new CalibrationOptions { ImportanceSampleRows = 0 }, 0); + } + + [Fact] + public void CorrectionSubtractsThePredictedError() + { + MzCalibrator calibrator = FitTrivialModel(out _); + var corrector = new SpectrumCorrector(calibrator, new CorrectionOptions()); + + var spectrum = new SpectrumRecord + { + MsLevel = 2, + PeakCount = 3, + MzArray = new[] { 400.0, 500.0, 600.0 }, + IntensityArray = new[] { 1000.0, 2000.0, 3000.0 }, + PrecursorMzLow = 399.5, + PrecursorMzHigh = 400.5, + PrecursorMzCenter = 400.0, + SummedIntensity = 6000.0, + InjectionTime = 0.02, + }; + + var corrected = new double[3]; + SpectrumCorrectionResult result = corrector.Correct(spectrum, null, new CorrectionWorkspace(), corrected); + + Assert.True(result.Corrected); + for (var i = 0; i < 3; i++) + { + // corrected = observed - predicted error, and the error here is a constant 0.02. + Assert.InRange(spectrum.MzArray[i] - corrected[i], 0.015, 0.025); + } + } + + [Fact] + public void ClampingKeepsTheArrayStrictlyAscending() + { + MzCalibrator calibrator = FitTrivialModel(out _); + var corrector = new SpectrumCorrector( + calibrator, new CorrectionOptions { Monotonicity = MonotonicityPolicy.ClampAscending }); + + // Two peaks a hair apart: any per-peak correction can reorder them. + var spectrum = new SpectrumRecord + { + MsLevel = 2, + PeakCount = 4, + MzArray = new[] { 500.0, 500.0000001, 500.0000002, 700.0 }, + IntensityArray = new[] { 100.0, 100.0, 100.0, 100.0 }, + PrecursorMzCenter = 500.0, + PrecursorMzLow = 499.5, + PrecursorMzHigh = 500.5, + SummedIntensity = 400.0, + InjectionTime = 0.02, + }; + + var corrected = new double[4]; + corrector.Correct(spectrum, null, new CorrectionWorkspace(), corrected); + + for (var i = 1; i < corrected.Length; i++) + { + Assert.True(corrected[i] > corrected[i - 1], + $"m/z array must stay strictly ascending: [{i - 1}]={corrected[i - 1]:R} [{i}]={corrected[i]:R}"); + } + } + + [Fact] + public void WideIsolationWindowsAreLeftAlone() + { + MzCalibrator calibrator = FitTrivialModel(out _); + var corrector = new SpectrumCorrector( + calibrator, new CorrectionOptions { MaxIsolationWindowWidth = 5.0 }); + + var spectrum = new SpectrumRecord + { + MsLevel = 2, + PeakCount = 2, + MzArray = new[] { 400.0, 500.0 }, + IntensityArray = new[] { 1000.0, 1000.0 }, + PrecursorMzLow = 400.0, + PrecursorMzHigh = 430.0, + PrecursorMzCenter = 415.0, + SummedIntensity = 2000.0, + }; + + var corrected = new double[2]; + SpectrumCorrectionResult result = corrector.Correct(spectrum, null, new CorrectionWorkspace(), corrected); + + Assert.False(result.Corrected); + Assert.Equal(spectrum.MzArray, corrected); + } + + [Fact] + public void Ms1SpectraAreNeverCorrected() + { + MzCalibrator calibrator = FitTrivialModel(out _); + var corrector = new SpectrumCorrector(calibrator, new CorrectionOptions()); + + var spectrum = new SpectrumRecord + { + MsLevel = 1, + PeakCount = 2, + MzArray = new[] { 400.0, 500.0 }, + IntensityArray = new[] { 1000.0, 1000.0 }, + SummedIntensity = 2000.0, + }; + + var corrected = new double[2]; + Assert.False(corrector.Correct(spectrum, null, new CorrectionWorkspace(), corrected).Corrected); + Assert.Equal(spectrum.MzArray, corrected); + } +} diff --git a/dotnet/MARS.Test/CommandLineArgsTest.cs b/dotnet/MARS.Test/CommandLineArgsTest.cs new file mode 100644 index 0000000..e6529f7 --- /dev/null +++ b/dotnet/MARS.Test/CommandLineArgsTest.cs @@ -0,0 +1,294 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.RegularExpressions; +using MARS.Cli; +using Xunit; + +namespace MARS.Test; + +public class CommandLineArgsTest +{ + [Fact] + public void AnOptionNoCommandAsksAboutIsUnknown() + { + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc", "--tolernace-ppm", "10" }); + args.Double("tolerance-ppm"); + + var ex = Assert.Throws(() => args.RejectUnknown()); + Assert.Contains("--tolernace-ppm", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void ANearMissSuggestsTheOptionThatWasMeant() + { + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc", "--tolernace-ppm", "10" }); + args.Double("tolerance-ppm"); + args.Double("tolerance"); + + var ex = Assert.Throws(() => args.RejectUnknown()); + Assert.Contains("Did you mean --tolerance-ppm?", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void SomethingUnrelatedSuggestsNothing() + { + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc", "--wombat" }); + args.Double("tolerance-ppm"); + args.Int("threads"); + + var ex = Assert.Throws(() => args.RejectUnknown()); + Assert.Contains("--wombat", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("Did you mean", ex.Message, StringComparison.Ordinal); + } + + /// + /// An option is recognized by being asked about, whether or not it was supplied - so + /// reading an absent option still teaches the parser that the name is valid. + /// + [Fact] + public void AskingAboutAnAbsentOptionStillRecognizesTheName() + { + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc", "--threads", "4" }); + args.Int("threads"); + args.Double("tolerance-ppm"); // absent, but now a name the command understands + + args.RejectUnknown(); + + CommandLineArgs supplied = CommandLineArgs.Parse(new[] { "qc", "--tolerance-ppm", "10" }); + supplied.Int("threads"); + supplied.Double("tolerance-ppm"); + supplied.RejectUnknown(); + } + + [Fact] + public void AliasesAreAllRecognizedNotJustTheOneThatMatched() + { + CommandLineArgs args = CommandLineArgs.Parse(new[] { "verify", "-i", "a.mzML" }); + args.String("input", "i"); + args.RejectUnknown(); + } + + /// + /// The check is only correct where it sits: an option a command reads after the check has + /// not been queried yet, so a misplaced call would reject a perfectly valid option. This + /// runs each command with every option its own help text documents and asserts none of + /// them is rejected. + /// + /// + /// The commands fail afterwards on missing inputs, which is fine and is the point - the + /// assertion is about which exception comes out, not about getting a successful run. + /// + [Theory] + [MemberData(nameof(DocumentedOptions))] + public void EveryDocumentedOptionSurvivesTheUnknownOptionCheck(string command, string[] options) + { + string directory = Path.Combine(Path.GetTempPath(), "mars-opts-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string mzml = Path.Combine(directory, "input.mzML"); + SyntheticMzML.Write(mzml, spectrumCount: 4, chromatogramCount: 0); + + // Each command names its input differently, and passing the wrong one would hand + // it an option it rightly does not know. + var argv = command switch + { + "verify" => new List { command, "--input", mzml }, + "compare" => new List { command, mzml, mzml }, + _ => new List { command, "--mzml", mzml }, + }; + argv.AddRange(options); + + Exception? thrown = Record.Exception(() => Run(command, argv.ToArray())); + + Assert.False( + thrown is UnknownOptionException, + $"mars {command} rejected one of its own documented options: {thrown?.Message}"); + } + finally + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } + } + + /// + /// A command that never runs the check accepts a typo in silence, which is worse than + /// rejecting a good option: the run completes, having quietly used a default for whatever + /// the user meant to set, and writes files that look fine. + /// + /// + /// This was true of apply, verify and compare - only calibrate and qc ran the check - so + /// the sibling test above was passing vacuously for them. + /// + [Theory] + [InlineData("qc")] + [InlineData("calibrate")] + [InlineData("apply")] + [InlineData("verify")] + [InlineData("compare")] + public void EveryCommandRejectsATypo(string command) + { + string directory = Path.Combine(Path.GetTempPath(), "mars-typo-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string mzml = Path.Combine(directory, "input.mzML"); + SyntheticMzML.Write(mzml, spectrumCount: 4, chromatogramCount: 0); + + var argv = command switch + { + "verify" => new List { command, "--input", mzml }, + "compare" => new List { command, mzml, mzml }, + _ => new List { command, "--mzml", mzml }, + }; + + // apply needs a model before it reads anything else, so give it one that exists. + if (command == "apply") + { + string model = Path.Combine(directory, "model.json"); + File.WriteAllText(model, "{}"); + argv.AddRange(new[] { "--model", model }); + } + + argv.AddRange(new[] { "--not-a-real-option", "7" }); + + Exception? thrown = Record.Exception(() => Run(command, argv.ToArray())); + + Assert.True( + thrown is UnknownOptionException, + $"mars {command} accepted --not-a-real-option; it threw {thrown?.GetType().Name ?? "nothing"}"); + } + finally + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } + } + + /// + /// The refusal has to reach the user as an error, not as an unhandled exception. It is + /// raised by throwing, so Program has to catch it - which it did not, briefly, and the + /// only symptom was a stack trace where a one-line message belonged. + /// + [Fact] + public void ATypoIsReportedAsAnInputErrorRatherThanACrash() + { + string directory = Path.Combine(Path.GetTempPath(), "mars-typo-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string mzml = Path.Combine(directory, "input.mzML"); + SyntheticMzML.Write(mzml, spectrumCount: 4, chromatogramCount: 0); + + int exit = Program.Main(new[] + { + "qc", "--mzml", mzml, "--prism-csv", "nonexistent.csv", "--tolernace", "0.3", + }); + + Assert.Equal(Program.ExitInputError, exit); + } + finally + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } + } + + private static void Run(string command, string[] argv) + { + CommandLineArgs args = CommandLineArgs.Parse(argv); + switch (command) + { + case "qc": QcCommand.Run(args); break; + case "calibrate": CalibrateCommand.Run(args); break; + case "apply": ApplyCommand.Run(args); break; + case "verify": VerifyCommand.Run(args); break; + case "compare": CompareCommand.Run(args); break; + default: throw new ArgumentOutOfRangeException(nameof(command), command, "Unknown command."); + } + } + + /// + /// Every option each command documents in its own --help, scraped from that help + /// text rather than listed here. + /// + /// + /// The first version of this was a hand-written array per command, and it drifted: a + /// `--resolution` option was added to the CLI and not to the list, so the check passed + /// while that option was in fact being rejected as a typo. Reading the help text means the + /// test cannot fall behind the thing it is testing - if an option is documented, it is + /// checked. + /// + public static TheoryData DocumentedOptions() + { + var data = new TheoryData(); + foreach (string command in new[] { "qc", "calibrate", "apply", "verify", "compare" }) + data.Add(command, OptionsFromHelp(command)); + return data; + } + + /// Pulls the long option names out of a command's help text. + private static string[] OptionsFromHelp(string command) + { + string help = CaptureHelp(command); + var options = new List(); + + foreach (Match match in Regex.Matches(help, @"--([a-z0-9][a-z0-9-]*)")) + { + string name = match.Groups[1].Value; + + // "--help" would print and exit, and the file arguments are supplied by the caller. + if (name is "help" or "mzml" or "input") continue; + if (options.Contains("--" + name)) continue; + + options.Add("--" + name); + + // A value for anything that takes one. The help text shows a placeholder in angle + // brackets after options that do; flags have nothing after them. + if (Regex.IsMatch(help, Regex.Escape("--" + name) + @"[= ]<")) + options.Add(ValueFor(name)); + } + + return options.ToArray(); + } + + /// A value each option will accept, so parsing gets far enough to matter. + private static string ValueFor(string name) => name switch + { + "resolution" => "auto", + "robust" => "trim", + "on-reorder" => "clamp", + "output-format" => "mzML", + var n when n.Contains("dir") => ".", + var n when n.Contains("csv") => "lib.csv", + var n when n.Contains("parquet") || n.Contains("report") => "report.parquet", + var n when n.Contains("library") => "lib.blib", + var n when n.Contains("model") => "model.json", + var n when n.Contains("path") || n.Contains("output") => "out.txt", + _ => "1", + }; + + private static string CaptureHelp(string command) + { + TextWriter original = Console.Error; + using var captured = new StringWriter(); + Console.SetError(captured); + try + { + Run(command, new[] { command, "--help" }); + } + catch (Exception) + { + // A command that refuses to run without inputs has still printed its help. + } + finally + { + Console.SetError(original); + } + + string help = captured.ToString(); + Assert.False(string.IsNullOrWhiteSpace(help), $"mars {command} --help printed nothing"); + return help; + } +} \ No newline at end of file diff --git a/dotnet/MARS.Test/CrossValidationTest.cs b/dotnet/MARS.Test/CrossValidationTest.cs new file mode 100644 index 0000000..4091dc8 --- /dev/null +++ b/dotnet/MARS.Test/CrossValidationTest.cs @@ -0,0 +1,492 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Linq; +using MARS.Core; +using Xunit; + +namespace MARS.Test; + +public sealed class PeptideFoldsTest +{ + private static int[] GroupsOf(params int[] groups) => groups; + + [Fact] + public void EveryRowOfAPeptideLandsInOneFold() + { + // 12 peptides, a different number of rows each, deliberately interleaved so a + // row-order split would scatter them. + var groupOfRow = new List(); + for (int row = 0; row < 600; row++) groupOfRow.Add(row % 12); + + (int[] foldOfRow, int groupCount) = PeptideFolds.AssignFolds(groupOfRow.ToArray(), folds: 5); + + Assert.Equal(12, groupCount); + + var foldOfGroup = new Dictionary(); + for (int row = 0; row < groupOfRow.Count; row++) + { + int group = groupOfRow[row]; + if (foldOfGroup.TryGetValue(group, out int fold)) + { + // This is the property the whole design rests on. A peptide split across + // folds lets the model memorize its fragment m/z values and report an + // accuracy it cannot reach on anything new. + Assert.Equal(fold, foldOfRow[row]); + } + else + { + foldOfGroup[group] = foldOfRow[row]; + } + } + } + + [Fact] + public void FoldsAreBalancedAndEveryFoldIsUsed() + { + var groupOfRow = new int[1000]; + for (int i = 0; i < groupOfRow.Length; i++) groupOfRow[i] = i / 10; // 100 peptides + + (int[] foldOfRow, _) = PeptideFolds.AssignFolds(groupOfRow, folds: 5); + + var groupsPerFold = new int[5]; + var seen = new HashSet(); + for (int row = 0; row < groupOfRow.Length; row++) + { + if (seen.Add(groupOfRow[row])) groupsPerFold[foldOfRow[row]]++; + } + + Assert.All(groupsPerFold, count => Assert.Equal(20, count)); + } + + [Fact] + public void AssignmentIsDeterministic() + { + var groupOfRow = new int[500]; + for (int i = 0; i < groupOfRow.Length; i++) groupOfRow[i] = (i * 7) % 53; + + (int[] first, _) = PeptideFolds.AssignFolds(groupOfRow, folds: 4); + (int[] second, _) = PeptideFolds.AssignFolds(groupOfRow, folds: 4); + + Assert.Equal(first, second); + } + + [Fact] + public void RefusesFewerThanTwoFolds() + { + Assert.Throws( + () => PeptideFolds.AssignFolds(GroupsOf(1, 2, 3), folds: 1)); + } + + [Fact] + public void HeldOutSplitKeepsPeptidesWhole() + { + var groupOfRow = new int[1000]; + for (int i = 0; i < groupOfRow.Length; i++) groupOfRow[i] = i / 10; + + (int[] train, int[] validation) = PeptideFolds.SplitByGroup(groupOfRow, 0.2, seed: 42); + + Assert.Equal(groupOfRow.Length, train.Length + validation.Length); + Assert.Empty(train.Intersect(validation)); + + var trainGroups = new HashSet(train.Select(r => groupOfRow[r])); + var validationGroups = new HashSet(validation.Select(r => groupOfRow[r])); + Assert.Empty(trainGroups.Intersect(validationGroups)); + + // Whole groups, so the fraction is approximate rather than exact. + Assert.InRange(validation.Length, 150, 250); + } + + [Fact] + public void HeldOutSplitOfZeroKeepsEverythingForTraining() + { + var groupOfRow = new int[100]; + for (int i = 0; i < groupOfRow.Length; i++) groupOfRow[i] = i / 5; + + (int[] train, int[] validation) = PeptideFolds.SplitByGroup(groupOfRow, 0, seed: 42); + + Assert.Equal(100, train.Length); + Assert.Empty(validation); + } + + [Fact] + public void MeasureReportsResidualAccuracy() + { + var observed = new double[] { 1.0, 2.0, 3.0, 4.0 }; + var predicted = new double[] { 1.1, 1.9, 3.1, 3.9 }; + + FoldMetrics metrics = PeptideFolds.Measure(observed, predicted); + + Assert.Equal(4, metrics.Rows); + Assert.Equal(0.1, metrics.Mad, 6); + Assert.Equal(0.1, metrics.Rms, 6); + + // Near 1 but not 1: the residuals alternate sign, so the predictions are not a + // linear transform of the observations. + Assert.Equal(0.9965, metrics.PearsonR, 4); + } + + [Fact] + public void CorrelationIsOneForAnyPositiveLinearTransform() + { + var observed = new double[] { 1.0, 2.0, 3.0, 4.0 }; + var predicted = new double[] { 0.5, 1.5, 2.5, 3.5 }; + + // Shifted by a constant, so the model has a bias but tracks the error perfectly. + // Pearson r cannot see the bias; that is what the median residual is for. + FoldMetrics metrics = PeptideFolds.Measure(observed, predicted); + Assert.Equal(1.0, metrics.PearsonR, 10); + Assert.Equal(0.5, metrics.Median, 10); + } + + [Fact] + public void PerfectPredictionLeavesNoResidual() + { + var observed = new double[] { -0.05, 0.02, 0.11, -0.2, 0.07 }; + FoldMetrics metrics = PeptideFolds.Measure(observed, observed); + + Assert.Equal(0.0, metrics.Mad, 12); + Assert.Equal(0.0, metrics.Rms, 12); + } + + [Fact] + public void AConstantPredictionHasNoCorrelationToReport() + { + var observed = new double[] { 1.0, 2.0, 3.0, 4.0 }; + var predicted = new double[] { 2.5, 2.5, 2.5, 2.5 }; + + // Not zero: a constant has no variance, so the correlation is undefined rather than + // absent. Reporting 0 would read as "measured, and there is no relationship". + Assert.True(double.IsNaN(PeptideFolds.Measure(observed, predicted).PearsonR)); + } + + [Fact] + public void SpreadAcrossFoldsIsTheSampleStandardDeviation() + { + FoldMetrics Fold(double mad) => new() + { + Rows = 100, Mad = mad, Rms = mad, StdDev = mad, Median = 0, + PearsonR = 0.5, MadBefore = 1.0, + }; + + var report = new CrossValidationReport + { + Folds = 3, + Groups = 30, + PerFold = new[] { Fold(0.10), Fold(0.12), Fold(0.14) }, + OutOfFold = Fold(0.12), + InSample = Fold(0.10), + }; + + Assert.Equal(0.02, report.MadSpread, 6); + Assert.Equal(0.02, report.OptimismMad, 6); + } +} + +public sealed class CrossValidatedFitTest +{ + private static MatchTable BuildMatches(int peptides = 200, int rowsEach = 10) + { + MarsFeature[] collect = + { + MarsFeature.PrecursorMz, MarsFeature.FragmentMz, + MarsFeature.LogTic, MarsFeature.LogIntensity, + }; + + var table = new MatchTable(collect); + var random = new Random(20260821); + + for (int p = 0; p < peptides; p++) + { + // One fragment m/z per peptide, repeated across spectra. This is what makes a + // row-random split leak: the same m/z would appear on both sides of it. + double fragmentMz = 200.0 + (random.NextDouble() * 1000.0); + for (int r = 0; r < rowsEach; r++) + { + double intensity = 500.0 + (random.NextDouble() * 100000.0); + table.Set(MarsFeature.PrecursorMz, 400.0 + (p % 20)); + table.Set(MarsFeature.FragmentMz, fragmentMz); + table.Set(MarsFeature.LogTic, Math.Log10(1.0e6)); + table.Set(MarsFeature.LogIntensity, Math.Log10(intensity)); + table.DeltaMz.Add((fragmentMz * 2.0e-5) - 0.01 + ((random.NextDouble() - 0.5) * 0.004)); + table.ObservedIntensity.Add(intensity); + table.PeptideGroup.Add(p); + table.CommitRow(); + } + } + + return table; + } + + [Fact] + public void ProducesOneModelPerFoldAndReportsEveryOne() + { + MzCalibrator calibrator = MzCalibrator.Fit( + BuildMatches(), new CalibrationOptions { CvFolds = 5, ImportanceSampleRows = 500 }, + absoluteTimeOffset: 0); + + CrossValidationReport cv = calibrator.CrossValidation!; + Assert.Equal(5, cv.Folds); + Assert.Equal(200, cv.Groups); + Assert.Equal(5, cv.PerFold.Length); + + // Every row got exactly one out-of-fold prediction, so the pooled count is the + // total. A fold that silently scored nothing would show up here. + Assert.Equal(2000, cv.OutOfFold.Rows); + Assert.Equal(2000, cv.PerFold.Sum(f => f.Rows)); + } + + [Fact] + public void TheAfterFiguresDescribeTheDataThatWasActuallyCorrected() + { + MzCalibrator calibrator = MzCalibrator.Fit( + BuildMatches(), new CalibrationOptions { CvFolds = 5, ImportanceSampleRows = 500 }, + absoluteTimeOffset: 0); + + CrossValidationReport cv = calibrator.CrossValidation!; + TrainingStatistics stats = calibrator.Statistics!; + + // The headline "after" figure describes the applied model on the rows it was fitted + // to, which is what the corrected files will look like when re-matched. Quoting the + // out-of-fold number here would understate what the correction achieved. + Assert.Equal(cv.InSample.Mad, stats.After.Mad, 12); + + // And the out-of-fold estimate must not be better than the in-sample one, or the + // arithmetic is wrong somewhere. + Assert.True(cv.InSample.Mad <= cv.OutOfFold.Mad + 1e-12, + $"in-sample {cv.InSample.Mad:R} should not be worse than out-of-fold {cv.OutOfFold.Mad:R}"); + } + + [Fact] + public void CrossValidationEstimatesWhatApplyWouldAchieveElsewhere() + { + MzCalibrator calibrator = MzCalibrator.Fit( + BuildMatches(), new CalibrationOptions { CvFolds = 5, ImportanceSampleRows = 0 }, + absoluteTimeOffset: 0); + + CrossValidationReport cv = calibrator.CrossValidation!; + + // Every row scored by a model that never saw its peptide, and every row scored + // exactly once. + Assert.Equal(2000, cv.OutOfFold.Rows); + Assert.True(cv.OptimismMad >= -1e-12, $"gap should not be negative: {cv.OptimismMad:R}"); + } + + [Fact] + public void SingleFitStillWorksAndCarriesNoCrossValidation() + { + MzCalibrator calibrator = MzCalibrator.Fit( + BuildMatches(), new CalibrationOptions { CvFolds = 0, ImportanceSampleRows = 500 }, + absoluteTimeOffset: 0); + + Assert.Null(calibrator.CrossValidation); + Assert.NotNull(calibrator.Statistics); + } + + [Fact] + public void TheAppliedModelIsAnOrdinarySingleFit() + { + MzCalibrator folded = MzCalibrator.Fit( + BuildMatches(), new CalibrationOptions { CvFolds = 5, NEstimators = 20, ImportanceSampleRows = 0 }, + absoluteTimeOffset: 0); + + MzCalibrator single = MzCalibrator.Fit( + BuildMatches(), new CalibrationOptions { CvFolds = 0, NEstimators = 20, ImportanceSampleRows = 0 }, + absoluteTimeOffset: 0); + + // Cross-validating does not change what gets applied: one model of the requested + // size, fitted to every row. The folds are a measurement taken alongside it, so + // correcting costs the same either way. + Assert.Equal(20, folded.Model.ToModelData().TreeRoot.Length); + Assert.Equal( + single.Model.ToModelData().TreeRoot.Length, + folded.Model.ToModelData().TreeRoot.Length); + } + + [Fact] + public void TwoIdenticalFitsProduceAByteIdenticalModelFile() + { + string Fit(int threads) + { + MzCalibrator calibrator = MzCalibrator.Fit( + BuildMatches(), + new CalibrationOptions { CvFolds = 5, MaxDegreeOfParallelism = threads, ImportanceSampleRows = 0 }, + absoluteTimeOffset: 0); + + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".json"); + try + { + MarsModelIo.Save(calibrator, path); + + // The version string is stamped from the assembly and the file is otherwise + // fully determined by the input, so hashing the whole thing is a fair test. + using var sha = SHA256.Create(); + return Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(path))); + } + finally + { + File.Delete(path); + } + } + + // Same input, same bytes - including across thread counts. MARS writes m/z values + // into files that get reprocessed and compared downstream, so a model that varied + // run to run would make every such comparison unreliable. + string first = Fit(1); + Assert.Equal(first, Fit(1)); + Assert.Equal(first, Fit(8)); + } + + [Fact] + public void RefusesToFoldFewerPeptidesThanFolds() + { + MatchTable table = BuildMatches(peptides: 3, rowsEach: 50); + + InvalidOperationException error = Assert.Throws( + () => MzCalibrator.Fit(table, new CalibrationOptions { CvFolds = 5 }, absoluteTimeOffset: 0)); + + Assert.Contains("cv-folds", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void RefusesATableMissingItsPeptideColumn() + { + MarsFeature[] collect = + { + MarsFeature.PrecursorMz, MarsFeature.FragmentMz, + MarsFeature.LogTic, MarsFeature.LogIntensity, + }; + + var table = new MatchTable(collect); + for (int i = 0; i < 100; i++) + { + table.Set(MarsFeature.PrecursorMz, 400.0); + table.Set(MarsFeature.FragmentMz, 600.0 + i); + table.Set(MarsFeature.LogTic, 6.0); + table.Set(MarsFeature.LogIntensity, 4.0); + table.DeltaMz.Add(0.01); + table.ObservedIntensity.Add(1000.0); + // PeptideGroup deliberately not filled. + table.CommitRow(); + } + + // Falling back to a row-random split here would silently report an accuracy the + // model cannot reach, which is worse than refusing. + InvalidOperationException error = Assert.Throws( + () => MzCalibrator.Fit(table, new CalibrationOptions(), absoluteTimeOffset: 0)); + + Assert.Contains("peptide group", error.Message, StringComparison.Ordinal); + } +} + +public sealed class ResidualTrimTest +{ + /// + /// A clean linear relationship, plus a contaminated minority whose label is unrelated to + /// its features - the shape a mismatched peak takes, where the recorded delta belongs to + /// some other ion. + /// + private static MatchTable BuildContaminated(double contaminatedFraction) + { + MarsFeature[] collect = + { + MarsFeature.PrecursorMz, MarsFeature.FragmentMz, + MarsFeature.LogTic, MarsFeature.LogIntensity, + }; + + var table = new MatchTable(collect); + var random = new Random(20260823); + const int peptides = 300, rowsEach = 10; + + for (int p = 0; p < peptides; p++) + { + double fragmentMz = 200.0 + (random.NextDouble() * 1000.0); + for (int r = 0; r < rowsEach; r++) + { + double intensity = 500.0 + (random.NextDouble() * 100000.0); + bool contaminated = random.NextDouble() < contaminatedFraction; + + table.Set(MarsFeature.PrecursorMz, 400.0 + (p % 20)); + table.Set(MarsFeature.FragmentMz, fragmentMz); + table.Set(MarsFeature.LogTic, Math.Log10(1.0e6)); + table.Set(MarsFeature.LogIntensity, Math.Log10(intensity)); + + double truth = (fragmentMz * 6.0e-5) - 0.02; + table.DeltaMz.Add(contaminated + // Uniform across the matching window: what you get when the most intense + // peak in the window was not the fragment. + ? (random.NextDouble() - 0.5) * 0.6 + : truth + ((random.NextDouble() - 0.5) * 0.004)); + + table.ObservedIntensity.Add(intensity); + table.PeptideGroup.Add(p); + table.CommitRow(); + } + } + + return table; + } + + [Fact] + public void TrimmingImprovesAccuracyOnContaminatedData() + { + var options = new CalibrationOptions { CvFolds = 5, ImportanceSampleRows = 0 }; + + MzCalibrator without = MzCalibrator.Fit( + BuildContaminated(0.15), new CalibrationOptions + { + CvFolds = 5, ImportanceSampleRows = 0, Robust = RobustFit.None, + }, + absoluteTimeOffset: 0); + + MzCalibrator with = MzCalibrator.Fit( + BuildContaminated(0.15), options, absoluteTimeOffset: 0); + + // Out-of-fold, and the held-out rows are scored in full either way - the contaminated + // ones included. So this is a real improvement, not the measurement getting easier. + Assert.True( + with.CrossValidation!.OutOfFold.Mad < without.CrossValidation!.OutOfFold.Mad, + $"trimmed {with.CrossValidation.OutOfFold.Mad:R} should beat " + + $"untrimmed {without.CrossValidation.OutOfFold.Mad:R}"); + } + + [Fact] + public void TrimmingIsOffWhenTheSigmaIsZero() + { + MzCalibrator a = MzCalibrator.Fit( + BuildContaminated(0.1), + new CalibrationOptions { CvFolds = 0, ImportanceSampleRows = 0, Robust = RobustFit.None }, + absoluteTimeOffset: 0); + + MzCalibrator b = MzCalibrator.Fit( + BuildContaminated(0.1), + new CalibrationOptions { CvFolds = 0, ImportanceSampleRows = 0, Robust = RobustFit.None }, + absoluteTimeOffset: 0); + + // Deterministic, and identical to itself: disabling the second pass must not leave + // any residue of it. + Assert.Equal(a.Statistics!.After.Mad, b.Statistics!.After.Mad, 12); + } + + [Fact] + public void CleanDataIsBarelyTrimmedAtAll() + { + MzCalibrator clean = MzCalibrator.Fit( + BuildContaminated(0.0), + new CalibrationOptions { CvFolds = 5, ImportanceSampleRows = 0 }, + absoluteTimeOffset: 0); + + MzCalibrator dirty = MzCalibrator.Fit( + BuildContaminated(0.2), + new CalibrationOptions { CvFolds = 5, ImportanceSampleRows = 0 }, + absoluteTimeOffset: 0); + + // The threshold is in robust sigma, so it adapts: clean data has a tight residual + // distribution and loses almost nothing, while contaminated data has a wide one and + // the tail is what gets cut. A fixed Th threshold would not do that. + Assert.True(clean.CrossValidation!.OutOfFold.Mad < dirty.CrossValidation!.OutOfFold.Mad); + } +} diff --git a/dotnet/MARS.Test/CsvReaderTest.cs b/dotnet/MARS.Test/CsvReaderTest.cs new file mode 100644 index 0000000..6da26aa --- /dev/null +++ b/dotnet/MARS.Test/CsvReaderTest.cs @@ -0,0 +1,137 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using MARS.IO; +using Xunit; + +namespace MARS.Test; + +public sealed class CsvReaderTest +{ + private static CsvReader Reader(string text) => new(new StringReader(text)); + + [Fact] + public void ReadsHeaderAndRows() + { + using CsvReader csv = Reader("a,b,c\n1,2,3\n4,5,6\n"); + + Assert.True(csv.ReadHeader()); + Assert.Equal(new[] { "a", "b", "c" }, csv.Header); + Assert.Equal(1, csv.ColumnIndex("b")); + Assert.Equal(-1, csv.ColumnIndex("missing")); + + Assert.True(csv.ReadRow()); + Assert.Equal("2", csv.Field(1)); + Assert.Equal(3, csv.IntField(2)); + + Assert.True(csv.ReadRow()); + Assert.Equal("4", csv.Field(0)); + Assert.False(csv.ReadRow()); + } + + [Fact] + public void HandlesQuotedFields() + { + // Skyline writes protein descriptions containing commas and quotes. + using CsvReader csv = Reader("name,note\n\"Smith, John\",\"he said \"\"hi\"\"\"\n"); + + Assert.True(csv.ReadHeader()); + Assert.True(csv.ReadRow()); + Assert.Equal("Smith, John", csv.Field(0)); + Assert.Equal("he said \"hi\"", csv.Field(1)); + } + + [Fact] + public void HandlesCrLfAndAMissingFinalNewline() + { + using CsvReader csv = Reader("a,b\r\n1,2\r\n3,4"); + + Assert.True(csv.ReadHeader()); + Assert.True(csv.ReadRow()); + Assert.Equal("2", csv.Field(1)); + Assert.True(csv.ReadRow()); + Assert.Equal("4", csv.Field(1)); + Assert.False(csv.ReadRow()); + } + + [Fact] + public void EmptyAndUnparseableFieldsBecomeSentinels() + { + using CsvReader csv = Reader("a,b,c\n,#N/A,7\n"); + + Assert.True(csv.ReadHeader()); + Assert.True(csv.ReadRow()); + Assert.Equal(string.Empty, csv.Field(0)); + + // Skyline writes #N/A for a value it could not compute; it must not become 0. + Assert.True(double.IsNaN(csv.DoubleField(0))); + Assert.True(double.IsNaN(csv.DoubleField(1))); + Assert.Equal(7, csv.IntField(2)); + Assert.Equal(5, csv.IntField(0, fallback: 5)); + } + + [Fact] + public void ReportsMissingRequiredColumns() + { + using CsvReader csv = Reader("a,b\n1,2\n"); + Assert.True(csv.ReadHeader()); + + Assert.Empty(csv.RequireColumns("a", "b")); + Assert.Equal(new[] { "c" }, csv.RequireColumns("a", "c")); + } +} + +public sealed class RunNameFilterTest +{ + [Fact] + public void MatchesOnBaseNameIgnoringExtensions() + { + var filter = new RunNameFilter(new[] { "Ste-2024-12-02_HeLa_GPFDIA_400-500_14.mzML" }); + + Assert.True(filter.Matches("Ste-2024-12-02_HeLa_GPFDIA_400-500_14.raw")); + Assert.True(filter.Matches("Ste-2024-12-02_HeLa_GPFDIA_400-500_14")); + Assert.False(filter.Matches("Ste-2024-12-02_HeLa_GPFDIA_900-1000_22.raw")); + } + + [Fact] + public void MatchesShortReplicateNamesBySubstring() + { + // Skyline replicate names are often just the distinctive part of the file name. + var filter = new RunNameFilter(new[] { "Ste-2024-12-02_HeLa_20msIIT_GPFDIA_400-500_14.mzML" }); + Assert.True(filter.Matches("400-500_14")); + } + + [Fact] + public void StripsCorrectedFileSuffixesSoOutputCanBeReQced() + { + // Re-running qc on {input}-mars.mzML has to match the same library rows. + var filter = new RunNameFilter(new[] { "run_07-mars.mzML" }); + Assert.True(filter.Matches("run_07.raw")); + } + + [Fact] + public void AnEmptyFilterMatchesEverything() + { + var filter = new RunNameFilter(Array.Empty()); + Assert.False(filter.Active); + Assert.True(filter.Matches("anything at all")); + } +} + +public sealed class FragmentAnnotationTest +{ + [Theory] + [InlineData("y7", 'y', 7)] + [InlineData("b12", 'b', 12)] + [InlineData("y5-H2O", 'y', 5)] + [InlineData("precursor", '?', 0)] + [InlineData("", '?', 0)] + [InlineData("z3", 'z', 3)] + public void ParsesSkylineFragmentAnnotations(string annotation, char expectedType, int expectedNumber) + { + (char ionType, int ionNumber) = PrismCsvLibraryReader.ParseFragmentIon(annotation); + Assert.Equal(expectedType, ionType); + Assert.Equal(expectedNumber, ionNumber); + } +} diff --git a/dotnet/MARS.Test/CultureTest.cs b/dotnet/MARS.Test/CultureTest.cs new file mode 100644 index 0000000..5c921bb --- /dev/null +++ b/dotnet/MARS.Test/CultureTest.cs @@ -0,0 +1,194 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Globalization; +using System.IO; +using System.Text.RegularExpressions; +using MARS.Core; +using MARS.IO; +using MARS.IO.Sqlite; +using MARS.Report; +using Xunit; + +namespace MARS.Test; + +/// +/// MARS must produce the same bytes on a machine whose locale writes decimals with a comma. +/// +/// +/// +/// This used to be guaranteed by InvariantGlobalization, which forces the whole runtime +/// to the invariant culture. Builds that carry a vendor reader have to relax it - the Thermo +/// SDK constructs CultureInfo("en-US") and throws when cultures are unavailable - which +/// hands CurrentCulture back to the operating system. These tests run under a +/// comma-decimal culture on purpose, so the guarantee is checked rather than assumed. +/// +/// +/// The failure mode is not cosmetic. A German CurrentCulture turns 653.835516 into +/// "653,835516" in an SVG coordinate or a JSON number, and turns the string "653.835516" read +/// out of a BiblioSpec library into 653835516. +/// +/// +public class CultureTest : IDisposable +{ + private readonly CultureInfo _original = CultureInfo.CurrentCulture; + + public CultureTest() + { + // German: decimal comma, point as the group separator - the arrangement most likely + // to turn a correct number into a different correct-looking number. + var german = new CultureInfo("de-DE"); + CultureInfo.CurrentCulture = german; + CultureInfo.CurrentUICulture = german; + } + + public void Dispose() + { + CultureInfo.CurrentCulture = _original; + CultureInfo.CurrentUICulture = _original; + GC.SuppressFinalize(this); + } + + /// + /// Confirms the test is actually testing something: under this culture, formatting without + /// an explicit provider really does produce a comma. + /// + [Fact] + public void TheCultureUnderTestWouldBreakNumbers() + { + Assert.Equal("653,84", 653.835516.ToString("0.00")); + Assert.True(double.TryParse("653.835516", out double parsed)); + Assert.NotEqual(653.835516, parsed); + } + + /// + /// SVG coordinates are read by a browser, not a person. A comma where a point belongs + /// makes the attribute invalid and the figure blank. + /// + [Fact] + public void TheQcReportWritesPointsNotCommas() + { + var data = BuildReportData(); + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".html"); + try + { + QcHtmlReport.Write( + path, data, statistics: null, + new MatchStatistics { SpectraSeen = 10, FragmentsMatched = 400 }, + new[] { "run.mzML" }, "0.3 Th", "26.1.0", + MarsStatistics.Summarize(data.ErrorBefore)); + + string html = File.ReadAllText(path); + + // Any attribute value holding a comma-decimal number, e.g. width="12,5". + Match bad = Regex.Match(html, "=\"[-0-9]+,[0-9]+\""); + Assert.False(bad.Success, $"comma-decimal number in the report: {bad.Value}"); + } + finally + { + File.Delete(path); + } + } + + /// + /// A model saved on one machine has to load on another. JSON numbers are point-decimal by + /// specification, so a comma is not merely unusual - it is a different document. + /// + [Fact] + public void AModelRoundTripsUnderAForeignCulture() + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".json"); + try + { + MzCalibrator calibrator = TrainTiny(); + MarsModelIo.Save(calibrator, path); + + string json = File.ReadAllText(path); + Assert.DoesNotContain(",\"", json.Replace("\",\"", string.Empty), StringComparison.Ordinal); + Assert.False( + Regex.IsMatch(json, @":\s*-?\d+,\d+"), + "a JSON number was written with a decimal comma"); + + MzCalibrator loaded = MarsModelIo.Load(path); + Assert.Equal(calibrator.Features.Count, loaded.Features.Count); + } + finally + { + File.Delete(path); + } + } + + /// + /// The bug this whole class exists for. A BiblioSpec library can store a number as SQLite + /// text, and that text is whatever wrote the library - nothing to do with the locale of + /// the machine reading it. Parsed under a German culture, "653.835516" becomes + /// 653,835,516: a fragment m/z six orders of magnitude wrong, and no error to show for it. + /// + [Fact] + public void ANumberStoredAsTextInALibraryParsesTheSameEverywhere() + { + byte[] bytes = System.Text.Encoding.UTF8.GetBytes("653.835516"); + SqliteValue value = SqliteValue.FromText(bytes, 0, bytes.Length, System.Text.Encoding.UTF8); + + Assert.Equal(653.835516, value.AsDouble(), 6); + } + + /// Integers stored as text are the same story, with the group separator. + [Fact] + public void AnIntegerStoredAsTextParsesTheSameEverywhere() + { + byte[] bytes = System.Text.Encoding.UTF8.GetBytes("1234567"); + SqliteValue value = SqliteValue.FromText(bytes, 0, bytes.Length, System.Text.Encoding.UTF8); + + Assert.Equal(1234567L, value.AsInteger()); + } + + private static QcHtmlReport.Data BuildReportData() + { + var random = new Random(11); + var before = new double[400]; + var rt = new double[before.Length]; + var mz = new double[before.Length]; + var feature = new double[before.Length]; + for (int i = 0; i < before.Length; i++) + { + before[i] = (random.NextDouble() - 0.5) * 0.3; + rt[i] = random.NextDouble() * 60; + mz[i] = 400 + (random.NextDouble() * 600); + feature[i] = random.NextDouble() * 10; + } + + return new QcHtmlReport.Data + { + ErrorBefore = before, + ErrorAfter = Array.Empty(), + RetentionTime = rt, + FragmentMz = mz, + Features = new[] { ("log_intensity", feature) }, + ImportanceNames = Array.Empty(), + Importance = Array.Empty(), + }; + } + + private static MzCalibrator TrainTiny() + { + var features = new[] { MarsFeature.FragmentMz, MarsFeature.LogIntensity }; + var table = new MatchTable(features); + var random = new Random(5); + + for (var i = 0; i < 400; i++) + { + double fragmentMz = 300 + (random.NextDouble() * 700); + double intensity = 500 + (random.NextDouble() * 100000); + + table.Set(MarsFeature.FragmentMz, fragmentMz); + table.Set(MarsFeature.LogIntensity, Math.Log10(intensity)); + table.DeltaMz.Add((fragmentMz * 2.0e-5) + ((random.NextDouble() - 0.5) * 0.01)); + table.ObservedIntensity.Add(intensity); + table.PeptideGroup.Add(i / 8); + table.CommitRow(); + } + + return MzCalibrator.Fit(table, new CalibrationOptions { CvFolds = 0 }, absoluteTimeOffset: 0); + } +} diff --git a/dotnet/MARS.Test/DiannParquetTest.cs b/dotnet/MARS.Test/DiannParquetTest.cs new file mode 100644 index 0000000..fc28e80 --- /dev/null +++ b/dotnet/MARS.Test/DiannParquetTest.cs @@ -0,0 +1,211 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// +// No DIA-NN output is available in this repository, so the reader is exercised against +// parquet files written by the test. That covers the column handling, the RT-window join +// and the error paths; it does NOT prove agreement with a real DIA-NN release, which is +// still outstanding. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using MARS.Core; +using MARS.IO; +using Parquet; +using Parquet.Data; +using Parquet.Schema; +using Xunit; + +namespace MARS.Test; + +public sealed class DiannParquetTest : IDisposable +{ + private readonly string _directory; + + public DiannParquetTest() + { + _directory = Path.Combine(Path.GetTempPath(), "mars-diann-" + Guid.NewGuid().ToString("N")[..12]); + Directory.CreateDirectory(_directory); + } + + public void Dispose() + { + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + } + } + + [Fact] + public void ReadsLibraryAndJoinsRtWindows() + { + string libraryPath = Path.Combine(_directory, "report-lib.parquet"); + string reportPath = Path.Combine(_directory, "report.parquet"); + + WriteLibrary(libraryPath, precursors: 3, fragmentsPerPrecursor: 4); + WriteReport(reportPath, new[] + { + ("PEPTIDEA2", "run_01", 10.0, 10.5), + ("PEPTIDEB2", "run_01", 20.0, 20.5), + ("PEPTIDEC2", "other_run", 30.0, 30.5), + }); + + SpectralLibrary library = DiannParquetLibraryReader.Load( + libraryPath, reportPath, new[] { "run_01.mzML" }); + + Assert.Equal(3, library.EntryCount); + Assert.Equal(12, library.FragmentCount); + + // The first two precursors are identified in run_01 and take its window; the third + // is only in another run, so it has none and matches at any retention time. + Assert.Equal(10.0, library.RtStart[0]); + Assert.Equal(10.5, library.RtEnd[0]); + Assert.Equal(20.0, library.RtStart[1]); + Assert.True(double.IsNaN(library.RtStart[2])); + + Assert.Equal(4, library.FragmentStart[1] - library.FragmentStart[0]); + Assert.All(library.FragmentMz, mz => Assert.True(mz > 0)); + } + + [Fact] + public void WindowsWidenAcrossRuns() + { + string libraryPath = Path.Combine(_directory, "report-lib.parquet"); + string reportPath = Path.Combine(_directory, "report.parquet"); + + WriteLibrary(libraryPath, precursors: 1, fragmentsPerPrecursor: 2); + WriteReport(reportPath, new[] + { + ("PEPTIDEA2", "run_01", 10.0, 10.5), + ("PEPTIDEA2", "run_02", 9.0, 11.5), + }); + + SpectralLibrary library = DiannParquetLibraryReader.Load( + libraryPath, reportPath, new[] { "run_01.mzML", "run_02.mzML" }); + + // A spectrum from either run has to fall inside the window, so it covers both. + Assert.Equal(9.0, library.RtStart[0]); + Assert.Equal(11.5, library.RtEnd[0]); + } + + [Fact] + public void AReportHandedInAsALibraryIsNamed() + { + string reportPath = Path.Combine(_directory, "report.parquet"); + WriteReport(reportPath, new[] { ("PEPTIDEA2", "run_01", 10.0, 10.5) }); + + InvalidDataException error = Assert.Throws( + () => DiannParquetLibraryReader.Load(reportPath, reportPath, Array.Empty())); + + // Passing report.parquet where report-lib.parquet belongs is the usual mistake, so + // the message has to say so rather than list missing column names. + Assert.Contains("report-lib.parquet", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void MissingReportIsExplained() + { + string libraryPath = Path.Combine(_directory, "report-lib.parquet"); + WriteLibrary(libraryPath, precursors: 1, fragmentsPerPrecursor: 1); + + FileNotFoundException error = Assert.Throws( + () => DiannParquetLibraryReader.Load(libraryPath, null, Array.Empty())); + + Assert.Contains("--diann-report", error.Message, StringComparison.Ordinal); + } + + private static void WriteLibrary(string path, int precursors, int fragmentsPerPrecursor) + { + var precursorId = new List(); + var modifiedSequence = new List(); + var precursorCharge = new List(); + var precursorMz = new List(); + var productMz = new List(); + var relativeIntensity = new List(); // DIA-NN writes this as float + var fragmentType = new List(); + var fragmentCharge = new List(); + var seriesNumber = new List(); + + for (var p = 0; p < precursors; p++) + { + string id = "PEPTIDE" + (char)('A' + p) + "2"; + for (var f = 0; f < fragmentsPerPrecursor; f++) + { + precursorId.Add(id); + modifiedSequence.Add("PEPTIDE" + (char)('A' + p)); + precursorCharge.Add(2); + precursorMz.Add(500.0 + p); + productMz.Add(300.0 + (p * 10) + f); + relativeIntensity.Add(1.0f - (f * 0.1f)); + fragmentType.Add(f % 2 == 0 ? "y" : "b"); + fragmentCharge.Add(1); + seriesNumber.Add(f + 3); + } + } + + var fields = new List + { + new DataField(DiannParquetLibraryReader.PrecursorIdColumn), + new DataField(DiannParquetLibraryReader.ModifiedSequenceColumn), + new DataField(DiannParquetLibraryReader.StrippedSequenceColumn), + new DataField(DiannParquetLibraryReader.PrecursorChargeColumn), + new DataField(DiannParquetLibraryReader.PrecursorMzColumn), + new DataField(DiannParquetLibraryReader.ProductMzColumn), + new DataField(DiannParquetLibraryReader.RelativeIntensityColumn), + new DataField(DiannParquetLibraryReader.FragmentTypeColumn), + new DataField(DiannParquetLibraryReader.FragmentChargeColumn), + new DataField(DiannParquetLibraryReader.FragmentSeriesNumberColumn), + }; + + var columns = new List + { + precursorId.ToArray(), + modifiedSequence.ToArray(), + modifiedSequence.ToArray(), + precursorCharge.ToArray(), + precursorMz.ToArray(), + productMz.ToArray(), + relativeIntensity.ToArray(), + fragmentType.ToArray(), + fragmentCharge.ToArray(), + seriesNumber.ToArray(), + }; + + WriteParquet(path, fields, columns); + } + + private static void WriteReport(string path, (string Id, string Run, double Start, double Stop)[] rows) + { + var fields = new List + { + new DataField(DiannParquetLibraryReader.PrecursorIdColumn), + new DataField(DiannParquetLibraryReader.RunColumn), + new DataField(DiannParquetLibraryReader.RtStartColumn), + new DataField(DiannParquetLibraryReader.RtStopColumn), + }; + + var columns = new List + { + rows.Select(r => r.Id).ToArray(), + rows.Select(r => r.Run).ToArray(), + rows.Select(r => r.Start).ToArray(), + rows.Select(r => r.Stop).ToArray(), + }; + + WriteParquet(path, fields, columns); + } + + private static void WriteParquet(string path, List fields, List columns) + { + var schema = new ParquetSchema(fields.Cast().ToArray()); + using Stream stream = File.Create(path); + using ParquetWriter writer = ParquetWriter.CreateAsync(schema, stream).GetAwaiter().GetResult(); + using ParquetRowGroupWriter group = writer.CreateRowGroup(); + + for (var i = 0; i < fields.Count; i++) + group.WriteColumnAsync(new DataColumn(fields[i], columns[i])).GetAwaiter().GetResult(); + } +} diff --git a/dotnet/MARS.Test/ErrorScaleTest.cs b/dotnet/MARS.Test/ErrorScaleTest.cs new file mode 100644 index 0000000..529b951 --- /dev/null +++ b/dotnet/MARS.Test/ErrorScaleTest.cs @@ -0,0 +1,193 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using System.Text.RegularExpressions; +using MARS.Core; +using MARS.Report; +using Xunit; + +namespace MARS.Test; + +public class ErrorScaleTest +{ + /// + /// The two arrays are converted row for row, so a short one is a bug rather than a set of + /// rows worth zero ppm - which is what it used to produce, and reads in a QC figure as a + /// perfectly calibrated fragment. + /// + [Fact] + public void AShortMzArrayIsRefusedRatherThanPaddedWithZeros() + { + var error = new[] { 0.001, 0.001, 0.001 }; + var mz = new[] { 500.0, 1000.0 }; + + ArgumentException thrown = + Assert.Throws(() => ErrorScale.Ppm.Convert(error, mz)); + Assert.Contains("one m/z per error", thrown.Message, StringComparison.Ordinal); + } + + /// + /// `mars qc` draws its report with no after-correction series and passes an empty array + /// for it. That is not a mismatch, and refusing it would break every qc HTML report. + /// + [Fact] + public void AnEmptyErrorSeriesIsNotAMismatch() + { + double[] converted = ErrorScale.Ppm.Convert(Array.Empty(), new[] { 500.0, 900.0 }); + Assert.Empty(converted); + } + + [Fact] + public void PpmConversionUsesEachRowsOwnMz() + { + var error = new[] { 0.001, 0.001 }; + var mz = new[] { 500.0, 1000.0 }; + + double[] ppm = ErrorScale.Ppm.Convert(error, mz); + + // The same absolute error is twice as many ppm at half the m/z. Dividing an aggregate + // by one nominal mass would report these two rows as identical. + Assert.Equal(2.0, ppm[0], 9); + Assert.Equal(1.0, ppm[1], 9); + } + + [Fact] + public void AZeroMzDoesNotProduceInfinity() + { + double[] ppm = ErrorScale.Ppm.Convert(new[] { 0.001 }, new[] { 0.0 }); + + Assert.Equal(0, ppm[0]); + Assert.False(double.IsInfinity(ppm[0]) || double.IsNaN(ppm[0])); + } + + [Fact] + public void TheThScaleLeavesValuesAlone() + { + var error = new[] { 0.08, -0.04 }; + Assert.Same(error, ErrorScale.Th.Convert(error, new[] { 500.0, 500.0 })); + } + + [Fact] + public void EachScaleFormatsToThePrecisionItsNumbersLiveAt() + { + // 0.0445 Th is the interesting figure on trap data; 1.90 ppm on high-resolution data. + // Two decimals would erase the first, four would pad the second with noise. + Assert.Equal("0.0445", ErrorScale.Th.Format(0.04452)); + Assert.Equal("1.90", ErrorScale.Ppm.Format(1.9012)); + } + + /// + /// The gap is out-of-fold minus in-sample: the correction does better on the data it was + /// fitted to, so the figure reads positive. Written the other way round it renders with a + /// minus sign and says the opposite of what it means. + /// + [Fact] + public void TheGapIsPositiveWhenTheFitDoesBetterOnItsOwnData() + { + string html = WriteWithCrossValidation(ErrorScale.Th); + + Match gap = Regex.Match(html, @"Gap (-?[\d.]+) Th"); + Assert.True(gap.Success, "no gap figure in the report"); + Assert.Equal(0.0020, double.Parse(gap.Groups[1].Value), 4); + Assert.Equal( + CrossValidation().OptimismMad, + double.Parse(gap.Groups[1].Value), + 4); + } + + [Fact] + public void AHighResolutionReportIsDrawnInPpmThroughout() + { + string html = WriteWithCrossValidation(ErrorScale.Ppm); + + Assert.Contains("mass error (ppm)", html, StringComparison.Ordinal); + Assert.Contains("MAD (ppm)", html, StringComparison.Ordinal); + Assert.Matches(@"Gap [\d.]+ ppm", html); + + // No stray Th axis or column left behind on a report that is meant to be in ppm. + Assert.DoesNotContain("mass error (Th)", html, StringComparison.Ordinal); + Assert.DoesNotContain("MAD (Th)", html, StringComparison.Ordinal); + } + + /// + /// A ppm spread has to come from the per-fold ppm figures. Each fold converts at its own + /// distribution of fragment m/z, so scaling the Th spread by any single factor is wrong. + /// + [Fact] + public void ThePpmSpreadComesFromThePpmFolds() + { + string html = WriteWithCrossValidation(ErrorScale.Ppm); + CrossValidationReport cv = CrossValidation(); + + double expected = CrossValidationReport.Spread(cv.PerFoldPpm!, static f => f.Mad); + Assert.Contains("+/-" + ErrorScale.Ppm.Format(expected), html, StringComparison.Ordinal); + Assert.NotEqual(cv.MadSpread, expected, 6); + } + + private static FoldMetrics Fold(double mad, double r) => new() + { + Rows = 100, Mad = mad, Rms = mad * 2, StdDev = mad * 2, + Median = 0, PearsonR = r, MadBefore = mad * 2, + }; + + private static CrossValidationReport CrossValidation() => new() + { + Folds = 3, + Groups = 60, + PerFold = new[] { Fold(0.0440, 0.69), Fold(0.0450, 0.68), Fold(0.0460, 0.70) }, + OutOfFold = Fold(0.0450, 0.69), + InSample = Fold(0.0430, 0.71), + + // Deliberately not a fixed multiple of the Th folds, so a report that derived ppm by + // scaling the Th spread would disagree with these numbers. + PerFoldPpm = new[] { Fold(1.80, 0.69), Fold(1.90, 0.68), Fold(2.30, 0.70) }, + OutOfFoldPpm = Fold(1.90, 0.69), + InSamplePpm = Fold(1.80, 0.71), + }; + + private static string WriteWithCrossValidation(ErrorScale scale) + { + var rows = new double[600]; + var mz = new double[rows.Length]; + var after = new double[rows.Length]; + var rt = new double[rows.Length]; + var feature = new double[rows.Length]; + var random = new Random(7); + for (int i = 0; i < rows.Length; i++) + { + rows[i] = (random.NextDouble() - 0.5) * 0.3; + after[i] = rows[i] * 0.5; + mz[i] = 400 + (random.NextDouble() * 600); + rt[i] = random.NextDouble() * 60; + feature[i] = random.NextDouble() * 10; + } + + var data = new QcHtmlReport.Data + { + ErrorBefore = rows, + ErrorAfter = after, + RetentionTime = rt, + FragmentMz = mz, + Features = new[] { ("log_intensity", feature) }, + ImportanceNames = new[] { "log_intensity" }, + Importance = new[] { 1.0 }, + CrossValidation = CrossValidation(), + }; + + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".html"); + try + { + QcHtmlReport.Write( + path, data, statistics: null, + new MatchStatistics { SpectraSeen = 10, FragmentsMatched = 600 }, + new[] { "run.mzML" }, scale.IsPpm ? "10 ppm" : "0.3 Th", "26.1.0", + uncorrected: null, scale); + return File.ReadAllText(path); + } + finally + { + File.Delete(path); + } + } +} diff --git a/dotnet/MARS.Test/FeatureExtractionTest.cs b/dotnet/MARS.Test/FeatureExtractionTest.cs new file mode 100644 index 0000000..ec2d2e1 --- /dev/null +++ b/dotnet/MARS.Test/FeatureExtractionTest.cs @@ -0,0 +1,289 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using MARS.Core; +using Xunit; + +namespace MARS.Test; + +public sealed class PeakSearchTest +{ + private static readonly double[] Mz = { 100.0, 100.4, 100.5, 101.0, 101.5, 102.0, 103.5, 104.0 }; + private static readonly double[] Intensity = { 10, 500, 300, 700, 200, 900, 50, 400 }; + + [Fact] + public void FindsTheMostIntensePeakNotTheClosest() + { + // 100.4 is nearer to the target than 100.5, but MARS takes the most intense peak in + // the window: a stronger peak has a better determined centroid. + bool found = PeakSearch.TryFindMostIntensePeak( + 100.45, Mz, Intensity, toleranceTh: 0.1, minIntensity: 0, tolerancePpm: 0, + out double mz, out double intensity); + + Assert.True(found); + Assert.Equal(100.4, mz); + Assert.Equal(500, intensity); + } + + [Fact] + public void TiesResolveToTheLowestMz() + { + var mz = new[] { 500.0, 500.1, 500.2 }; + var intensity = new[] { 100.0, 100.0, 100.0 }; + + bool found = PeakSearch.TryFindMostIntensePeak( + 500.1, mz, intensity, 0.5, 0, 0, out double best, out _); + + Assert.True(found); + Assert.Equal(500.0, best); + } + + [Fact] + public void MinimumIntensityFiltersCandidates() + { + bool found = PeakSearch.TryFindMostIntensePeak( + 100.2, Mz, Intensity, toleranceTh: 0.25, minIntensity: 600, tolerancePpm: 0, + out _, out _); + Assert.False(found); + + Assert.True(PeakSearch.TryFindMostIntensePeak( + 100.2, Mz, Intensity, 0.25, 400, 0, out double mz, out _)); + Assert.Equal(100.4, mz); + } + + [Fact] + public void PpmToleranceOverridesAbsoluteTolerance() + { + var mz = new[] { 999.99, 1000.0, 1000.02 }; + var intensity = new[] { 100.0, 50.0, 400.0 }; + + // 10 ppm at 1000 Th is 0.01 Th, so 1000.02 is out of range even though it is the + // most intense peak. + Assert.True(PeakSearch.TryFindMostIntensePeak( + 1000.0, mz, intensity, toleranceTh: 5.0, minIntensity: 0, tolerancePpm: 10, + out double found, out _)); + Assert.Equal(999.99, found); + } + + [Fact] + public void RangeSumIsExclusiveLowAndInclusiveHigh() + { + // (100.4, 101.0] holds 100.5 and 101.0 but not 100.4. + double sum = PeakSearch.SumIntensityInRange(Mz, Intensity, 100.4, 101.0); + Assert.Equal(300 + 700, sum); + + Assert.Equal(0.0, PeakSearch.SumIntensityInRange(Mz, Intensity, 105.0, 106.0)); + } + + /// + /// The sweep used when correcting must produce exactly what the per-fragment binary + /// search produces when training, or training and inference would see different values + /// for the same feature. + /// + [Fact] + public void SweepMatchesPerPeakRangeSums() + { + var random = new Random(4242); + var mz = new double[400]; + var intensity = new double[400]; + double value = 200.0; + for (var i = 0; i < mz.Length; i++) + { + value += 0.05 + (random.NextDouble() * 3.0); + mz[i] = value; + intensity[i] = random.NextDouble() * 10000.0; + } + + foreach ((double low, double high) in MarsFeatures.NeighborWindows) + { + var swept = new double[mz.Length]; + PeakSearch.ComputeNeighborWindow(mz, intensity, low, high, swept); + + for (var i = 0; i < mz.Length; i++) + { + double expected = PeakSearch.SumIntensityInRange(mz, intensity, mz[i] + low, mz[i] + high); + Assert.Equal(expected, swept[i]); + } + } + } +} + +public sealed class StatisticsTest +{ + [Fact] + public void MedianInterpolatesForEvenCounts() + { + Assert.Equal(2.5, MarsStatistics.Median(new[] { 1.0, 2.0, 3.0, 4.0 })); + Assert.Equal(3.0, MarsStatistics.Median(new[] { 1.0, 2.0, 3.0, 4.0, 100.0 })); + } + + [Fact] + public void MedianAbsoluteDeviationIsAboutTheMedian() + { + // median is 3; absolute deviations are 2,1,0,1,2 so the MAD is 1. + Assert.Equal(1.0, MarsStatistics.MedianAbsoluteDeviation(new[] { 1.0, 2.0, 3.0, 4.0, 5.0 })); + } + + [Fact] + public void StandardDeviationUsesTheSampleConvention() + { + // ddof = 1, matching pandas Series.std, which the Python report used. + double[] values = { 2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0 }; + Assert.Equal(2.13809, MarsStatistics.StdDev(values), 5); + } + + [Fact] + public void SummaryReportsEveryScale() + { + double[] values = { -0.1, 0.0, 0.1, 0.2 }; + ErrorSummary summary = MarsStatistics.Summarize(values); + + Assert.Equal(4, summary.Count); + Assert.Equal(0.05, summary.Mean, 12); + Assert.Equal(0.05, summary.Median, 12); + Assert.Equal(0.1, summary.Mae, 12); + } +} + +public sealed class PeptideMassTest +{ + /// + /// Reference values for the y and b series of a peptide, computed from the standard + /// monoisotopic residue masses. + /// + [Fact] + public void FragmentMzMatchesKnownValues() + { + const string peptide = "LLQDANYNVEK"; + + // Singly protonated y ions. + Assert.Equal(147.11280, PeptideMass.FragmentMz(peptide, 'y', 1, 1), 4); + Assert.Equal(276.15539, PeptideMass.FragmentMz(peptide, 'y', 2, 1), 4); + + // Singly protonated b ions. + Assert.Equal(227.17540, PeptideMass.FragmentMz(peptide, 'b', 2, 1), 4); + + // The precursor: neutral peptide plus two protons over two charges. + double neutral = 0; + foreach (char residue in peptide) neutral += PeptideMass.Residue(residue); + neutral += PeptideMass.Water; + double doubly = (neutral + (2 * PeptideMass.Proton)) / 2; + Assert.Equal(653.8355, doubly, 3); + } + + [Fact] + public void ChargeTwoHalvesTheNeutralMass() + { + double singly = PeptideMass.FragmentMz("PEPTIDEK", 'y', 4, 1); + double doubly = PeptideMass.FragmentMz("PEPTIDEK", 'y', 4, 2); + Assert.Equal((singly + PeptideMass.Proton) / 2, doubly, 9); + } + + [Fact] + public void ModificationsApplyOnlyInsideTheFragment() + { + const string peptide = "ACDEFGHIK"; + var carbamidomethyl = new List<(int, double)> { (2, 57.021464) }; + + // C is residue 2, so it is inside b3 but outside y3. + double b3Plain = PeptideMass.FragmentMz(peptide, 'b', 3, 1); + double b3Modified = PeptideMass.FragmentMz(peptide, 'b', 3, 1, carbamidomethyl); + Assert.Equal(b3Plain + 57.021464, b3Modified, 9); + + double y3Plain = PeptideMass.FragmentMz(peptide, 'y', 3, 1); + double y3Modified = PeptideMass.FragmentMz(peptide, 'y', 3, 1, carbamidomethyl); + Assert.Equal(y3Plain, y3Modified, 9); + } + + [Fact] + public void ModifiedSequencesSplitIntoResiduesAndDeltas() + { + (string stripped, List<(int Position, double Mass)> modifications, int unweighed) = + PeptideMass.SplitModifiedSequence("LSC[+57.021464]AASGFTFSSYAM[+15.994915]SWVR"); + + Assert.Equal("LSCAASGFTFSSYAMSWVR", stripped); + Assert.Equal(2, modifications.Count); + Assert.Equal(3, modifications[0].Position); + Assert.Equal(57.021464, modifications[0].Mass, 9); + Assert.Equal(15, modifications[1].Position); + Assert.Equal(0, unweighed); + } + + /// + /// A modification named rather than weighed has no mass here, and saying so is the whole + /// point: dropping it silently leaves the residue at its unmodified mass, and every + /// theoretical fragment past that position comes out wrong by the delta while looking + /// perfectly reasonable. + /// + [Theory] + [InlineData("LSC[Carbamidomethyl (C)]AASGFTFSSYAMSWVR", 1)] + [InlineData("LSCAASGFTFSSYAM(unimod:35)SWVR", 1)] + [InlineData("LSC[Carbamidomethyl (C)]AASGFTFSSYAM[+15.994915]SWVR", 1)] + public void ANamedModificationIsCountedRatherThanDropped(string sequence, int expected) + { + (string stripped, List<(int Position, double Mass)> modifications, int unweighed) = + PeptideMass.SplitModifiedSequence(sequence); + + Assert.Equal("LSCAASGFTFSSYAMSWVR", stripped); + Assert.Equal(expected, unweighed); + + // The named one contributes no delta; a numeric one alongside it still does. + Assert.DoesNotContain(modifications, m => m.Position == 3); + } + + [Fact] + public void UnknownResiduesYieldNaNRatherThanAWrongMass() + { + Assert.True(double.IsNaN(PeptideMass.FragmentMz("PEPXIDE", 'y', 5, 1))); + Assert.True(double.IsNaN(PeptideMass.FragmentMz("PEPTIDE", 'y', 99, 1))); + Assert.True(double.IsNaN(PeptideMass.FragmentMz("PEPTIDE", 'q', 3, 1))); + } +} + +public sealed class FeatureSetTest +{ + [Fact] + public void FeatureNamesAreTheOnDiskContract() + { + // These strings are what a model file records; changing one silently invalidates + // every model written before the change. + Assert.Equal("precursor_mz", MarsFeatures.NameOf(MarsFeature.PrecursorMz)); + Assert.Equal("ions_above_0_1", MarsFeatures.NameOf(MarsFeature.IonsAbove01)); + Assert.Equal("adjacent_ratio_below_2_3", MarsFeatures.NameOf(MarsFeature.AdjacentRatioBelow23)); + Assert.Equal(MarsFeatures.Count, MarsFeatures.Names.Length); + } + + [Fact] + public void NeighborWindowsUseTheDocumentedHalfThOffsets() + { + // Windows sit half a Th off the isotope spacing so each one centres on an isotope + // peak rather than straddling two. + Assert.Equal((0.5, 1.5), MarsFeatures.NeighborWindows[0]); + Assert.Equal((-1.5, -0.5), MarsFeatures.NeighborWindows[3]); + Assert.Equal(MarsFeatures.NeighborFeatures.Length, MarsFeatures.RatioFeatures.Length); + Assert.Equal(MarsFeatures.NeighborWindows.Length, MarsFeatures.NeighborFeatures.Length); + } + + [Fact] + public void SlotLookupFollowsTheDeclaredOrder() + { + var set = new FeatureSet(new[] { MarsFeature.FragmentMz, MarsFeature.LogTic, MarsFeature.Rfa2Temp }); + + Assert.Equal(0, set.SlotOf(MarsFeature.FragmentMz)); + Assert.Equal(2, set.SlotOf(MarsFeature.Rfa2Temp)); + Assert.Equal(-1, set.SlotOf(MarsFeature.PrecursorMz)); + Assert.False(set.NeedsNeighborDensity); + + var withNeighbors = new FeatureSet(new[] { MarsFeature.FragmentMz, MarsFeature.IonsAbove12 }); + Assert.True(withNeighbors.NeedsNeighborDensity); + } + + [Fact] + public void UnknownFeatureNameIsRejected() + { + Assert.Throws(() => FeatureSet.FromNames(new[] { "precursor_mz", "not_a_feature" })); + Assert.Throws(() => + new FeatureSet(new[] { MarsFeature.LogTic, MarsFeature.LogTic })); + } +} diff --git a/dotnet/MARS.Test/InjectionTimeTest.cs b/dotnet/MARS.Test/InjectionTimeTest.cs new file mode 100644 index 0000000..f18039d --- /dev/null +++ b/dotnet/MARS.Test/InjectionTimeTest.cs @@ -0,0 +1,203 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using System.Linq; +using MARS.Cli; +using MARS.Core; +using MARS.IO; +using Xunit; + +namespace MARS.Test; + +/// +/// Ion injection time is only a feature when it varies - but the features scaled by it are +/// features whenever it exists. +/// +/// +/// +/// A trap sets it per spectrum from its automatic gain control, so it says how full the trap +/// was. A Bruker or Sciex TOF accumulates for a fixed period, so every spectrum carries the +/// same number: injection_time becomes a constant, which a tree can never split on, and +/// tic_injection_time becomes TIC times that constant - log_tic rescaled, and a +/// duplicate that splits permutation importance with the feature it duplicates. +/// +/// +/// The ion-population features are a separate question, and getting the two confused was +/// expensive. They are peak sums over m/z windows, multiplied by the injection time to turn a +/// rate into a count. A constant injection time scales them all by the same factor, which +/// leaves every one of them varying and every split available. Dropping them alongside the +/// injection time took the Stellar reference cohort from 18 features to 5 and its corrected +/// MAD from 0.0463 Th to 0.0581 - on the instrument MARS was written for. +/// +/// +public class InjectionTimeTest +{ + [Fact] + public void AVaryingInjectionTimeIsUsed() => + Assert.Equal(InjectionTimeUse.Varying, Probe(constant: false)); + + [Fact] + public void AConstantInjectionTimeIsNotUsed() => + Assert.Equal(InjectionTimeUse.Constant, Probe(constant: true)); + + /// + /// Collection is not the decision. A run that records an injection time gets every column + /// that depends on one, and whether the injection time itself earns a place is settled + /// later from the whole column. + /// + [Theory] + [InlineData(InjectionTimeUse.Varying)] + [InlineData(InjectionTimeUse.Constant)] + public void ARunThatRecordsAnInjectionTimeCollectsEveryColumnThatNeedsOne(InjectionTimeUse use) + { + MarsFeature[] features = FragmentMatcher.CollectedFeatures(use, rfa2: false, rfc2: false); + + Assert.Contains(MarsFeature.InjectionTime, features); + Assert.Contains(MarsFeature.TicInjectionTime, features); + Assert.Contains(MarsFeature.FragmentIons, features); + foreach (MarsFeature f in MarsFeatures.NeighborFeatures) Assert.Contains(f, features); + foreach (MarsFeature f in MarsFeatures.RatioFeatures) Assert.Contains(f, features); + } + + /// + /// A column that is flat over its first few hundred rows and moves later has to read as + /// varying. This is the case that was wrong, and it is not a corner case: an ion trap sits + /// at the method's ceiling for the entire void volume, so every real gradient looks like + /// this. Judged on its head, a standard Stellar DIA run reads as constant while two thirds + /// of its spectra are off the ceiling. + /// + [Fact] + public void AColumnThatOnlyMovesLaterInTheRunStillVaries() + { + var table = new MatchTable(new[] { MarsFeature.InjectionTime }); + + for (var i = 0; i < 5000; i++) + { + // Flat for the first 4,000 rows, then the trap starts filling. + table.Set(MarsFeature.InjectionTime, i < 4000 ? 10.0 : 10.0 - ((i - 4000) * 0.001)); + Row(table); + } + + Assert.True(table.Varies(MarsFeature.InjectionTime)); + } + + [Fact] + public void AGenuinelyConstantColumnDoesNotVary() + { + var table = new MatchTable(new[] { MarsFeature.InjectionTime }); + for (var i = 0; i < 1000; i++) + { + table.Set(MarsFeature.InjectionTime, 10.672768592834); + Row(table); + } + + Assert.False(table.Varies(MarsFeature.InjectionTime)); + } + + /// A column of nothing does not vary, and must not be reported as though it did. + [Fact] + public void AnAbsentColumnDoesNotVary() + { + var table = new MatchTable(new[] { MarsFeature.InjectionTime }); + for (var i = 0; i < 10; i++) + { + table.Set(MarsFeature.InjectionTime, double.NaN); + Row(table); + } + + Assert.False(table.Varies(MarsFeature.InjectionTime)); + Assert.False(table.AnyFinite(MarsFeature.InjectionTime)); + } + + /// + /// End to end through the fit: a constant injection time drops itself and + /// tic_injection_time, and takes nothing else with it. + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void OnlyTheInjectionTimeItselfLeavesWhenItIsConstant(bool constant) + { + MarsFeature[] collect = + { + MarsFeature.PrecursorMz, MarsFeature.FragmentMz, MarsFeature.LogTic, + MarsFeature.LogIntensity, MarsFeature.InjectionTime, MarsFeature.TicInjectionTime, + MarsFeature.FragmentIons, + }; + + var table = new MatchTable(collect); + var random = new Random(11); + + for (var i = 0; i < 2000; i++) + { + double injection = constant ? 10.0 : 6.0 + (random.NextDouble() * 4.0); + double intensity = 500 + (random.NextDouble() * 100000); + + table.Set(MarsFeature.PrecursorMz, 400 + (random.NextDouble() * 500)); + table.Set(MarsFeature.FragmentMz, 300 + (random.NextDouble() * 700)); + table.Set(MarsFeature.LogTic, 6.0 + random.NextDouble()); + table.Set(MarsFeature.LogIntensity, Math.Log10(intensity)); + table.Set(MarsFeature.InjectionTime, injection); + table.Set(MarsFeature.TicInjectionTime, 1e6 * injection); + table.Set(MarsFeature.FragmentIons, intensity * injection); + + table.DeltaMz.Add(0.01 + (random.NextDouble() * 0.01)); + table.ObservedIntensity.Add(intensity); + table.PeptideGroup.Add(i / 8); + table.CommitRow(); + } + + MzCalibrator calibrator = MzCalibrator.Fit( + table, new CalibrationOptions { CvFolds = 0, ImportanceSampleRows = 0 }, 0); + + Assert.Equal(!constant, calibrator.Features.Contains(MarsFeature.InjectionTime)); + Assert.Equal(!constant, calibrator.Features.Contains(MarsFeature.TicInjectionTime)); + + // The one that must survive either way. + Assert.True(calibrator.Features.Contains(MarsFeature.FragmentIons)); + } + + private static void Row(MatchTable table) + { + table.DeltaMz.Add(0.01); + table.ObservedIntensity.Add(1000); + table.PeptideGroup.Add(0); + table.CommitRow(); + } + + /// + /// With no injection time at all they do have to go: there is nothing to turn a rate into + /// a count with, and the matcher yields NaN for every one of them. + /// + [Fact] + public void TheIonPopulationFeaturesNeedAnInjectionTimeToExist() + { + MarsFeature[] features = + FragmentMatcher.CollectedFeatures(InjectionTimeUse.Absent, rfa2: false, rfc2: false); + + Assert.DoesNotContain(MarsFeature.FragmentIons, features); + foreach (MarsFeature f in MarsFeatures.NeighborFeatures) Assert.DoesNotContain(f, features); + foreach (MarsFeature f in MarsFeatures.RatioFeatures) Assert.DoesNotContain(f, features); + } + + private static InjectionTimeUse Probe(bool constant) + { + string directory = Path.Combine(Path.GetTempPath(), "mars-inj-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "input.mzML"); + SyntheticMzML.Write( + path, spectrumCount: 40, chromatogramCount: 0, peaksPerSpectrum: 6, + constantInjectionTime: constant); + + using var source = new MzMLSpectrumSource(path); + return CalibrateCommand.ProbeInjectionTime(source); + } + finally + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } + } +} diff --git a/dotnet/MARS.Test/MARS.Test.csproj b/dotnet/MARS.Test/MARS.Test.csproj new file mode 100644 index 0000000..135ab87 --- /dev/null +++ b/dotnet/MARS.Test/MARS.Test.csproj @@ -0,0 +1,44 @@ + + + + + + + + + + + + false + MARS.Test + MARS.Test + false + true + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + diff --git a/dotnet/MARS.Test/MassAnalyzerTest.cs b/dotnet/MARS.Test/MassAnalyzerTest.cs new file mode 100644 index 0000000..35dbddc --- /dev/null +++ b/dotnet/MARS.Test/MassAnalyzerTest.cs @@ -0,0 +1,343 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using MARS.Cli; +using MARS.Core; +using MARS.IO; +using Xunit; + +namespace MARS.Test; + +public class MassAnalyzerTest +{ + [Theory] + [InlineData(MassAnalyzers.Orbitrap, MassAnalyzerClass.HighResolution)] + [InlineData(MassAnalyzers.AsymmetricTrackLosslessTimeOfFlight, MassAnalyzerClass.HighResolution)] + [InlineData(MassAnalyzers.TimeOfFlight, MassAnalyzerClass.HighResolution)] + [InlineData(MassAnalyzers.FourierTransformIonCyclotronResonance, MassAnalyzerClass.HighResolution)] + [InlineData(MassAnalyzers.RadialEjectionLinearIonTrap, MassAnalyzerClass.UnitResolution)] + [InlineData(MassAnalyzers.QuadrupoleIonTrap, MassAnalyzerClass.UnitResolution)] + [InlineData(MassAnalyzers.Quadrupole, MassAnalyzerClass.UnitResolution)] + [InlineData("MS:9999999", MassAnalyzerClass.Unknown)] + [InlineData(null, MassAnalyzerClass.Unknown)] + public void AccessionsClassify(string? accession, MassAnalyzerClass expected) => + Assert.Equal(expected, MassAnalyzers.Classify(accession)); + + [Theory] + [InlineData("ITMS + c NSI t Full ms2 601.02@hcd30.00", MassAnalyzerClass.UnitResolution)] + [InlineData("FTMS + c NSI Full ms [375.0000-985.0000]", MassAnalyzerClass.HighResolution)] + [InlineData("ASTMS + c NSI Full ms2 413.93@hcd27.00", MassAnalyzerClass.HighResolution)] + [InlineData("something else entirely", MassAnalyzerClass.Unknown)] + [InlineData("", MassAnalyzerClass.Unknown)] + public void FilterStringsClassify(string filter, MassAnalyzerClass expected) => + Assert.Equal(expected, MassAnalyzers.ClassifyFilterString(filter)); + + /// + /// The quadrupole in a hybrid configuration isolates rather than measures. Choosing it + /// would call an Astral run unit-resolution and set a tolerance two orders of magnitude + /// too wide. + /// + [Fact] + public void TheMeasuringAnalyzerIsNotTheIsolatingQuadrupole() + { + string? measuring = MassAnalyzers.MeasuringAnalyzer(new[] + { + (2, MassAnalyzers.Quadrupole), + (3, MassAnalyzers.AsymmetricTrackLosslessTimeOfFlight), + }); + + Assert.Equal(MassAnalyzers.AsymmetricTrackLosslessTimeOfFlight, measuring); + } + + [Fact] + public void AQuadrupoleOnlyConfigurationStillReportsTheQuadrupole() + { + string? measuring = MassAnalyzers.MeasuringAnalyzer(new[] { (2, MassAnalyzers.Quadrupole) }); + Assert.Equal(MassAnalyzers.Quadrupole, measuring); + } + + /// Order decides among analyzers, not the order they happen to be listed in. + [Fact] + public void TheHighestOrderAnalyzerWinsWhicheverWayRoundTheyAreListed() + { + var forwards = new[] { (2, MassAnalyzers.IonTrap), (3, MassAnalyzers.Orbitrap) }; + var backwards = new[] { (3, MassAnalyzers.Orbitrap), (2, MassAnalyzers.IonTrap) }; + + Assert.Equal(MassAnalyzers.Orbitrap, MassAnalyzers.MeasuringAnalyzer(forwards)); + Assert.Equal(MassAnalyzers.Orbitrap, MassAnalyzers.MeasuringAnalyzer(backwards)); + } + + [Fact] + public void ATrapFileIsReadAsUnitResolution() => + Assert.Equal( + MassAnalyzerClass.UnitResolution, + Detect(SyntheticMzML.MassAnalyzerLayout.UnitResolutionTrap)); + + /// + /// The case the whole mechanism exists for. The run names the orbitrap as its default + /// because that is what takes the MS1 survey; only the MS2 spectra point at the Astral + /// analyzer, and MS2 is what MARS calibrates. + /// + [Fact] + public void AHybridFileIsClassifiedByItsMs2AnalyzerNotItsRunDefault() => + Assert.Equal( + MassAnalyzerClass.HighResolution, + Detect(SyntheticMzML.MassAnalyzerLayout.HybridOrbitrapAstral)); + + [Fact] + public void AFileThatDoesNotSayIsUnknownRatherThanGuessed() => + Assert.Equal(MassAnalyzerClass.Unknown, Detect(SyntheticMzML.MassAnalyzerLayout.None)); + + /// + /// A cohort spanning two kinds of instrument gets one tolerance, so it has to be said out + /// loud. The tolerance is set from the first file; every other file is measured against it. + /// + [Fact] + public void AMixedCohortIsReported() + { + var analyzers = new Dictionary + { + ["stellar.mzML"] = MassAnalyzerClass.UnitResolution, + ["astral.mzML"] = MassAnalyzerClass.HighResolution, + }; + + string? odd = CalibrateCommand.FirstAnalyzerDisagreement( + analyzers.Keys.ToList(), f => analyzers[f], MassAnalyzerClass.UnitResolution); + + Assert.Equal("astral.mzML", odd); + } + + [Fact] + public void ACohortOnOneKindOfInstrumentIsNotReported() + { + var analyzers = new Dictionary + { + ["a.mzML"] = MassAnalyzerClass.HighResolution, + ["b.mzML"] = MassAnalyzerClass.HighResolution, + }; + + Assert.Null(CalibrateCommand.FirstAnalyzerDisagreement( + analyzers.Keys.ToList(), f => analyzers[f], MassAnalyzerClass.HighResolution)); + } + + /// + /// A file that does not name its analyzer is not evidence of a mixed cohort. Warning on + /// it would fire on most mzML in existence, and a warning that always fires is noise. + /// + [Fact] + public void AFileThatDoesNotSayIsNotADisagreement() + { + var analyzers = new Dictionary + { + ["known.mzML"] = MassAnalyzerClass.HighResolution, + ["silent.mzML"] = MassAnalyzerClass.Unknown, + }; + + Assert.Null(CalibrateCommand.FirstAnalyzerDisagreement( + analyzers.Keys.ToList(), f => analyzers[f], MassAnalyzerClass.HighResolution)); + + // And with nothing detected at all there is nothing to compare against. + Assert.Null(CalibrateCommand.FirstAnalyzerDisagreement( + analyzers.Keys.ToList(), f => analyzers[f], MassAnalyzerClass.Unknown)); + } + + /// + /// Adding the configuration list must not disturb a fixture without one - every other + /// test in the suite reads those bytes. + /// + [Fact] + public void TheDefaultFixtureIsUnchangedByTheNewParameter() + { + string directory = NewDirectory(); + try + { + string implicitDefault = Path.Combine(directory, "implicit.mzML"); + string explicitNone = Path.Combine(directory, "explicit.mzML"); + SyntheticMzML.Write(implicitDefault, spectrumCount: 8, chromatogramCount: 1); + SyntheticMzML.Write( + explicitNone, spectrumCount: 8, chromatogramCount: 1, + analyzers: SyntheticMzML.MassAnalyzerLayout.None); + + Assert.Equal(File.ReadAllBytes(implicitDefault), File.ReadAllBytes(explicitNone)); + } + finally + { + Delete(directory); + } + } + + /// + /// Detection sets a default; it never overrules the person at the terminal, who can be + /// sure in a way a heuristic cannot. + /// + [Fact] + public void AnExplicitToleranceBeatsDetection() + { + string directory = NewDirectory(); + try + { + string path = Path.Combine(directory, "astral.mzML"); + SyntheticMzML.Write( + path, spectrumCount: 8, chromatogramCount: 0, + analyzers: SyntheticMzML.MassAnalyzerLayout.HybridOrbitrapAstral); + + var options = new MatchOptions(); + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc", "--tolerance", "0.5" }); + options.MzToleranceTh = args.Double("tolerance") ?? ResolutionMode.DefaultToleranceTh; + + ResolutionMode mode = ResolutionMode.Resolve(args, Detect(path), options, _ => { }); + + Assert.Equal(MassAnalyzerClass.HighResolution, mode.Analyzer); + Assert.Equal(0.5, options.MzToleranceTh); + Assert.Equal(0, options.TolerancePpm); + } + finally + { + Delete(directory); + } + } + + [Fact] + public void DetectionPicksThePpmToleranceOnHighResolutionData() + { + string directory = NewDirectory(); + try + { + string path = Path.Combine(directory, "astral.mzML"); + SyntheticMzML.Write( + path, spectrumCount: 8, chromatogramCount: 0, + analyzers: SyntheticMzML.MassAnalyzerLayout.HybridOrbitrapAstral); + + var options = new MatchOptions(); + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc" }); + ResolutionMode mode = ResolutionMode.Resolve(args, Detect(path), options, _ => { }); + + Assert.True(mode.ReportInPpm); + Assert.Equal(ResolutionMode.DefaultTolerancePpm, options.TolerancePpm); + Assert.Equal(0, options.MzToleranceTh); + } + finally + { + Delete(directory); + } + } + + /// + /// --resolution overrides what the file says, for data whose header is wrong or absent. + /// + [Fact] + public void AnExplicitModeOverridesWhatTheFileSays() + { + string directory = NewDirectory(); + try + { + string path = Path.Combine(directory, "trap.mzML"); + SyntheticMzML.Write( + path, spectrumCount: 8, chromatogramCount: 0, + analyzers: SyntheticMzML.MassAnalyzerLayout.UnitResolutionTrap); + + var options = new MatchOptions(); + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc", "--resolution", "hram" }); + ResolutionMode mode = ResolutionMode.Resolve(args, Detect(path), options, _ => { }); + + Assert.Equal(MassAnalyzerClass.HighResolution, mode.Analyzer); + Assert.Equal(ResolutionMode.DefaultTolerancePpm, options.TolerancePpm); + } + finally + { + Delete(directory); + } + } + + [Fact] + public void AnUnrecognizedModeIsRejected() + { + CommandLineArgs args = CommandLineArgs.Parse(new[] { "qc", "--resolution", "sideways" }); + Assert.Throws( + () => ResolutionMode.Resolve(args, MassAnalyzerClass.Unknown, new MatchOptions(), _ => { })); + } + + /// + /// mars verify writes a round-tripped copy and deletes it unless --keep, so an --output + /// pointing at the input would destroy the file the command exists to vouch for. + /// + [Fact] + public void VerifyRefusesToWriteOverItsInput() + { + string directory = NewDirectory(); + try + { + string path = Path.Combine(directory, "run.mzML"); + SyntheticMzML.Write(path, spectrumCount: 8, chromatogramCount: 0); + long before = new FileInfo(path).Length; + + int exit = VerifyCommand.Run( + CommandLineArgs.Parse(new[] { "verify", "--input", path, "--output", path })); + + Assert.Equal(Program.ExitInputError, exit); + Assert.True(File.Exists(path), "verify deleted its own input"); + Assert.Equal(before, new FileInfo(path).Length); + } + finally + { + Delete(directory); + } + } + + /// The same file reached by a different spelling of the path is still the same file. + [Fact] + public void VerifyRefusesAnInputAndOutputThatOnlyLookDifferent() + { + string directory = NewDirectory(); + try + { + string path = Path.Combine(directory, "run.mzML"); + SyntheticMzML.Write(path, spectrumCount: 8, chromatogramCount: 0); + string roundabout = Path.Combine(directory, ".", "run.mzML"); + + int exit = VerifyCommand.Run( + CommandLineArgs.Parse(new[] { "verify", "--input", path, "--output", roundabout })); + + Assert.Equal(Program.ExitInputError, exit); + Assert.True(File.Exists(path)); + } + finally + { + Delete(directory); + } + } + + /// What a reader would detect for a file already on disk. + private static MassAnalyzerClass Detect(string path) => + MzMLFile.DetectMs2Analyzer(MzMLFile.Inspect(path)); + + private static MassAnalyzerClass Detect(SyntheticMzML.MassAnalyzerLayout layout) + { + string directory = NewDirectory(); + try + { + string path = Path.Combine(directory, "input.mzML"); + SyntheticMzML.Write(path, spectrumCount: 12, chromatogramCount: 0, analyzers: layout); + return MzMLFile.DetectMs2Analyzer(MzMLFile.Inspect(path)); + } + finally + { + Delete(directory); + } + } + + private static string NewDirectory() + { + string directory = Path.Combine(Path.GetTempPath(), "mars-analyzer-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + return directory; + } + + private static void Delete(string directory) + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } +} diff --git a/dotnet/MARS.Test/MatchDumpTest.cs b/dotnet/MARS.Test/MatchDumpTest.cs new file mode 100644 index 0000000..d8ed3fb --- /dev/null +++ b/dotnet/MARS.Test/MatchDumpTest.cs @@ -0,0 +1,178 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Globalization; +using System.IO; +using MARS.Core; +using MARS.IO; +using Xunit; + +namespace MARS.Test; + +public sealed class MatchDumpTest +{ + private static SpectralLibrary BuildLibrary() + { + var builder = new SpectralLibraryBuilder(keepSequences: true); + builder.BeginEntry("PEPTIDER", 2, 500.25, 10.0, 11.0); + builder.AddFragment(600.30, 1000, 'y', 5, 1); + builder.AddFragment(700.40, 2000, 'b', 3, 2); + builder.BeginEntry("SEQ[+80.0]UENCE", 3, 400.10, 12.0, 13.0); + builder.AddFragment(800.50, 3000, 'y', 7, 1); + return builder.Build(); + } + + /// + /// A predictions array that is not parallel to the table is refused before the file is + /// opened, rather than throwing partway through millions of rows with a half-written dump + /// on disk and nothing naming the cause. + /// + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(0)] + public void AMismatchedPredictionsArrayIsRefusedUpFront(int predictionCount) + { + string directory = Path.Combine(Path.GetTempPath(), "mars-dump-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "dump.csv"); + MatchTable table = BuildTable(); + Assert.NotEqual(predictionCount, table.Count); + + ArgumentException error = Assert.Throws(() => + MatchDumpWriter.Write(path, table, BuildLibrary(), new double[predictionCount])); + + Assert.Contains("parallel", error.Message, StringComparison.Ordinal); + Assert.False(File.Exists(path), "no file should be left behind"); + } + finally + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } + } + + private static MatchTable BuildTable() + { + MarsFeature[] collect = { MarsFeature.PrecursorMz, MarsFeature.FragmentMz, MarsFeature.LogIntensity }; + var table = new MatchTable(collect, keepDetail: true); + + AddRow(table, scan: 101, retentionTime: 10.5, entry: 0, fragment: 0, + observedMz: 600.35, deltaMz: 0.05, observedIntensity: 1234.5, + precursorMz: 500.25, fragmentMz: 600.30, logIntensity: 3.09); + AddRow(table, scan: 102, retentionTime: 12.5, entry: 1, fragment: 2, + observedMz: 800.44, deltaMz: -0.06, observedIntensity: 987.0, + precursorMz: 400.10, fragmentMz: 800.50, logIntensity: double.NaN); + + return table; + } + + private static void AddRow( + MatchTable table, int scan, double retentionTime, int entry, int fragment, + double observedMz, double deltaMz, double observedIntensity, + double precursorMz, double fragmentMz, double logIntensity) + { + table.Set(MarsFeature.PrecursorMz, precursorMz); + table.Set(MarsFeature.FragmentMz, fragmentMz); + table.Set(MarsFeature.LogIntensity, logIntensity); + table.DeltaMz.Add(deltaMz); + table.ObservedIntensity.Add(observedIntensity); + table.PeptideGroup.Add(entry); + table.ScanNumber!.Add(scan); + table.LibraryEntryIndex!.Add(entry); + table.FragmentIndex!.Add(fragment); + table.ObservedMz!.Add(observedMz); + table.RetentionTime!.Add(retentionTime); + table.CommitRow(); + } + + [Fact] + public void WritesOneRowPerMatchWithIdentityAndFeatures() + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".csv"); + try + { + MatchDumpWriter.Write(path, BuildTable(), BuildLibrary()); + string[] lines = File.ReadAllLines(path); + + Assert.Equal(3, lines.Length); + Assert.Equal( + "scan_number,retention_time,entry_index,fragment_index,peptide_group,peptide,ion_annotation," + + "expected_mz,observed_mz,delta_mz,observed_intensity," + + "precursor_mz,fragment_mz,log_intensity", + lines[0]); + + string[] first = lines[1].Split(','); + Assert.Equal("101", first[0]); + Assert.Equal("0", first[2]); + Assert.Equal("0", first[3]); + Assert.Equal("0", first[4]); + Assert.Equal("\"PEPTIDER\"", first[5]); + Assert.Equal("y5+1", first[6]); + Assert.Equal(600.30, double.Parse(first[7], CultureInfo.InvariantCulture), 6); + Assert.Equal(600.35, double.Parse(first[8], CultureInfo.InvariantCulture), 6); + + // The second row points at the second entry's only fragment, which is index 2 in + // the flat arrays. Getting this wrong would silently label rows with another + // peptide's identity, so it is asserted rather than assumed. + string[] second = lines[2].Split(','); + Assert.Equal("\"SEQ[+80.0]UENCE\"", second[5]); + Assert.Equal("y7+1", second[6]); + Assert.Equal(800.50, double.Parse(second[7], CultureInfo.InvariantCulture), 6); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void RoundTripsDoublesWithoutLosingPrecision() + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".csv"); + try + { + MatchDumpWriter.Write(path, BuildTable(), BuildLibrary()); + string[] columns = File.ReadAllLines(path)[1].Split(','); + + // "R" formatting is what makes a dump usable as a comparison oracle: a value that + // rounds on the way out cannot be differenced against another implementation at + // the precision that matters for m/z. + Assert.Equal(1234.5, double.Parse(columns[10], CultureInfo.InvariantCulture)); + Assert.Equal(500.25, double.Parse(columns[11], CultureInfo.InvariantCulture)); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void WritesNaNRatherThanBlankingIt() + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".csv"); + try + { + MatchDumpWriter.Write(path, BuildTable(), BuildLibrary()); + string[] columns = File.ReadAllLines(path)[2].Split(','); + + // NaN is how an undefined feature reaches the model, and row selection drops on + // it. A blank would read as "missing column" instead. + Assert.Equal("NaN", columns[13]); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void RefusesATableWithoutDetailColumns() + { + var table = new MatchTable(new[] { MarsFeature.FragmentMz }); + InvalidOperationException error = Assert.Throws( + () => MatchDumpWriter.Write("unused.csv", table, BuildLibrary())); + Assert.Contains("keepDetail", error.Message, StringComparison.Ordinal); + } +} diff --git a/dotnet/MARS.Test/MzMLComparerTest.cs b/dotnet/MARS.Test/MzMLComparerTest.cs new file mode 100644 index 0000000..bd738a3 --- /dev/null +++ b/dotnet/MARS.Test/MzMLComparerTest.cs @@ -0,0 +1,77 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using MARS.IO; +using Xunit; + +namespace MARS.Test; + +/// +/// What mars compare reports, and what it does when the two files stop lining up. +/// +/// +/// The comparison pairs spectra by position and checks the ids agree. That is right for what +/// it is for - a file against a correction of itself - but it cannot realign, so a file with a +/// spectrum inserted or removed would otherwise have every subsequent pair compared against +/// the wrong spectrum and counted as a difference. A tool whose job is to say "these files +/// agree" must not answer "they disagree everywhere" when they differ by one spectrum. +/// +public sealed class MzMLComparerTest : IDisposable +{ + private readonly string _directory; + + public MzMLComparerTest() + { + _directory = Path.Combine(Path.GetTempPath(), "mars-cmp-" + Guid.NewGuid().ToString("N")[..12]); + Directory.CreateDirectory(_directory); + } + + public void Dispose() + { + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + // A locked temp file must not fail the suite. + } + } + + [Fact] + public void IdenticalFilesCompareEqualAndDoNotDiverge() + { + string a = Path.Combine(_directory, "a.mzML"); + string b = Path.Combine(_directory, "b.mzML"); + SyntheticMzML.Write(a, spectrumCount: 12, chromatogramCount: 0); + SyntheticMzML.Write(b, spectrumCount: 12, chromatogramCount: 0); + + MzMLComparison result = MzMLComparer.Compare(a, b); + + Assert.False(result.Diverged); + Assert.Equal(12, result.SpectraCompared); + Assert.Equal(0, result.MzValuesDiffering); + Assert.Equal(0, result.SpectraOnlyInA); + Assert.Equal(0, result.SpectraOnlyInB); + } + + /// + /// Different spectrum counts leave the shorter file exhausted first, which is a plain + /// difference in length rather than a loss of alignment. + /// + [Fact] + public void AShorterFileIsCountedNotMisreported() + { + string a = Path.Combine(_directory, "long.mzML"); + string b = Path.Combine(_directory, "short.mzML"); + SyntheticMzML.Write(a, spectrumCount: 12, chromatogramCount: 0); + SyntheticMzML.Write(b, spectrumCount: 8, chromatogramCount: 0); + + MzMLComparison result = MzMLComparer.Compare(a, b); + + Assert.Equal(8, result.SpectraCompared); + Assert.Equal(4, result.SpectraOnlyInA); + Assert.Equal(0, result.MzValuesDiffering); + } +} diff --git a/dotnet/MARS.Test/MzMLPassthroughTest.cs b/dotnet/MARS.Test/MzMLPassthroughTest.cs new file mode 100644 index 0000000..0cd4b85 --- /dev/null +++ b/dotnet/MARS.Test/MzMLPassthroughTest.cs @@ -0,0 +1,225 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// The passthrough contract, exercised on a synthetic mzML built in the test itself so the +// suite needs no data files. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using MARS.Core; +using MARS.IO; +using Xunit; + +namespace MARS.Test; + +public sealed class MzMLPassthroughTest : IDisposable +{ + private readonly string _directory; + + public MzMLPassthroughTest() + { + _directory = Path.Combine(Path.GetTempPath(), "mars-test-" + Guid.NewGuid().ToString("N")[..12]); + Directory.CreateDirectory(_directory); + } + + public void Dispose() + { + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + // A locked temp file must not fail the suite. + } + } + + [Fact] + public void NullCorrectionRoundTripsBitIdentically() + { + string input = Path.Combine(_directory, "input.mzML"); + SyntheticMzML.Write(input, spectrumCount: 12, chromatogramCount: 2); + + string output = Path.Combine(_directory, "output.mzML"); + MzMLFileInfo info = MzMLFile.Inspect(input); + Assert.True(info.IsIndexedMzML); + Assert.True(info.WasIndexed); + + MzMLWriteResult result = MzMLWriter.Write(info, output, () => new NullMzTransform()); + + Assert.Equal(12, result.SpectraSeen); + Assert.Equal(2, result.ChromatogramsCopied); + Assert.True(result.WroteIndex); + + MzMLComparison comparison = MzMLComparer.Compare(input, output); + Assert.Equal(0, comparison.SpectraOnlyInA); + Assert.Equal(0, comparison.SpectraOnlyInB); + Assert.True(comparison.MzBitIdentical, string.Join("; ", comparison.Problems)); + Assert.True(comparison.IntensityBitIdentical); + } + + [Fact] + public void RegeneratedIndexAndChecksumValidate() + { + string input = Path.Combine(_directory, "input.mzML"); + SyntheticMzML.Write(input, spectrumCount: 20, chromatogramCount: 1); + + string output = Path.Combine(_directory, "output.mzML"); + MzMLWriter.Write(MzMLFile.Inspect(input), output, () => new NullMzTransform()); + + IndexValidationResult validation = MzMLValidator.Validate(output); + Assert.True(validation.IsIndexed); + Assert.Equal(20, validation.SpectrumOffsets); + Assert.Equal(1, validation.ChromatogramOffsets); + Assert.Empty(validation.BadOffsets); + Assert.True(validation.ChecksumPresent); + Assert.True(validation.ChecksumValid, + $"recorded {validation.RecordedChecksum}, computed {validation.ComputedChecksum}"); + } + + /// + /// Only the m/z array of a selected spectrum may change. The intensity array, the + /// metadata and every other byte have to survive untouched. + /// + [Fact] + public void CorrectionTouchesOnlyTheMzArray() + { + string input = Path.Combine(_directory, "input.mzML"); + SyntheticMzML.Write(input, spectrumCount: 8, chromatogramCount: 0); + + string output = Path.Combine(_directory, "output.mzML"); + MzMLWriter.Write(MzMLFile.Inspect(input), output, () => new ShiftTransform(0.01)); + + MzMLComparison comparison = MzMLComparer.Compare(input, output); + Assert.True(comparison.IntensityBitIdentical); + Assert.False(comparison.MzBitIdentical); + Assert.Equal(comparison.MzValuesCompared, comparison.MzValuesDiffering); + Assert.InRange(comparison.MaxAbsoluteMzDifference, 0.0099, 0.0101); + + // Structural checks still pass after a real modification. + IndexValidationResult validation = MzMLValidator.Validate(output); + Assert.Empty(validation.BadOffsets); + Assert.True(validation.ChecksumValid); + } + + [Fact] + public void OutputIsDeterministicAcrossThreadCounts() + { + string input = Path.Combine(_directory, "input.mzML"); + SyntheticMzML.Write(input, spectrumCount: 30, chromatogramCount: 1); + + string single = Path.Combine(_directory, "t1.mzML"); + string many = Path.Combine(_directory, "t16.mzML"); + + MzMLWriter.Write(MzMLFile.Inspect(input), single, () => new ShiftTransform(0.003), + new MzMLWriteOptions { MaxDegreeOfParallelism = 1 }); + MzMLWriter.Write(MzMLFile.Inspect(input), many, () => new ShiftTransform(0.003), + new MzMLWriteOptions { MaxDegreeOfParallelism = 16 }); + + // Inference has no cross-row accumulation, so thread count cannot change a value: + // assert on file bytes, which is the strongest form of the guarantee. + Assert.Equal(File.ReadAllBytes(single), File.ReadAllBytes(many)); + } + + [Fact] + public void EncodingIsPreservedPerArray() + { + // 64-bit zlib m/z beside 32-bit uncompressed intensity is the case that catches a + // writer that reads encoding once per spectrum instead of once per array. + string input = Path.Combine(_directory, "mixed.mzML"); + SyntheticMzML.Write(input, spectrumCount: 6, chromatogramCount: 0, + mzEncoding: new BinaryArrayEncoding(true, true), + intensityEncoding: new BinaryArrayEncoding(false, false)); + + string output = Path.Combine(_directory, "mixed-out.mzML"); + MzMLWriter.Write(MzMLFile.Inspect(input), output, () => new NullMzTransform()); + + string text = File.ReadAllText(output); + Assert.Contains("MS:1000523", text); // 64-bit float still declared + Assert.Contains("MS:1000521", text); // 32-bit float still declared + Assert.Contains("MS:1000574", text); // zlib still declared + + MzMLComparison comparison = MzMLComparer.Compare(input, output); + Assert.True(comparison.MzBitIdentical); + Assert.True(comparison.IntensityBitIdentical); + } + + /// + /// Regression: a payload larger than the decompressor's internal buffer must still + /// decode. Inflating in place used to corrupt the compressed bytes still waiting to be + /// read, which no small spectrum could ever surface -- the whole payload fit in one + /// buffered read, so the aliasing never mattered until an array got big. + /// + [Fact] + public void LargeCompressedArraysDecodeCorrectly() + { + string input = Path.Combine(_directory, "large.mzML"); + SyntheticMzML.Write(input, spectrumCount: 3, chromatogramCount: 0, peaksPerSpectrum: 20000); + + // Well past the 8 KB the decompressor buffers internally. + var info = MzMLFile.Inspect(input); + var peakCounts = new List(); + foreach (SpectrumRecord spectrum in MzMLFile.ReadSpectra(info, msLevel: null)) + peakCounts.Add(spectrum.PeakCount); + + Assert.Equal(3, peakCounts.Count); + Assert.All(peakCounts, count => Assert.Equal(20000, count)); + + string output = Path.Combine(_directory, "large-out.mzML"); + MzMLWriter.Write(info, output, () => new NullMzTransform()); + + MzMLComparison comparison = MzMLComparer.Compare(input, output); + Assert.Equal(60000, comparison.MzValuesCompared); + Assert.True(comparison.MzBitIdentical, string.Join("; ", comparison.Problems)); + Assert.True(comparison.IntensityBitIdentical); + } + + [Fact] + public void EncodedLengthMatchesTheNewPayload() + { + string input = Path.Combine(_directory, "input.mzML"); + SyntheticMzML.Write(input, spectrumCount: 5, chromatogramCount: 0); + + string output = Path.Combine(_directory, "output.mzML"); + MzMLWriter.Write(MzMLFile.Inspect(input), output, () => new ShiftTransform(0.05)); + + string text = File.ReadAllText(output); + var checkedAny = false; + var cursor = 0; + while (true) + { + int at = text.IndexOf("encodedLength=\"", cursor, StringComparison.Ordinal); + if (at < 0) break; + + int start = at + "encodedLength=\"".Length; + int end = text.IndexOf('"', start); + int declared = int.Parse(text[start..end], CultureInfo.InvariantCulture); + + int binaryOpen = text.IndexOf("", end, StringComparison.Ordinal); + int binaryClose = text.IndexOf("", end, StringComparison.Ordinal); + if (binaryOpen < 0 || binaryClose < 0) break; + + int actual = binaryClose - (binaryOpen + "".Length); + Assert.Equal(declared, actual); + checkedAny = true; + cursor = binaryClose; + } + + Assert.True(checkedAny, "no binary arrays were checked"); + } + + private sealed class ShiftTransform : IMzTransform + { + private readonly double _shift; + + public ShiftTransform(double shift) => _shift = shift; + + public MzTransformResult Transform(SpectrumRecord spectrum, Span corrected) + { + ReadOnlySpan mz = spectrum.Mz; + for (var i = 0; i < mz.Length; i++) corrected[i] = mz[i] + _shift; + return new MzTransformResult { Rewrite = true }; + } + } +} diff --git a/dotnet/MARS.Test/MzMLWriterThreadsTest.cs b/dotnet/MARS.Test/MzMLWriterThreadsTest.cs new file mode 100644 index 0000000..ad996a4 --- /dev/null +++ b/dotnet/MARS.Test/MzMLWriterThreadsTest.cs @@ -0,0 +1,113 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using MARS.Core; +using MARS.IO; +using Xunit; + +namespace MARS.Test; + +/// +/// --threads has to bound the mzML write path, not just be accepted by it. +/// +/// +/// The correction is per-spectrum with no cross-row state, so the thread count cannot change +/// the output - which is exactly why an unbounded write is invisible. It shows up only as CPU +/// use, and the person who asked for one core is usually sharing the machine with someone. +/// +public sealed class MzMLWriterThreadsTest : IDisposable +{ + private readonly string _directory; + + public MzMLWriterThreadsTest() + { + _directory = Path.Combine(Path.GetTempPath(), "mars-test-" + Guid.NewGuid().ToString("N")[..12]); + Directory.CreateDirectory(_directory); + } + + public void Dispose() + { + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + // A locked temp file must not fail the suite. + } + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public void NoMoreSpectraAreInFlightThanThreadsAllows(int threads) + { + string input = Path.Combine(_directory, "input.mzML"); + SyntheticMzML.Write(input, spectrumCount: 200, chromatogramCount: 0); + + var counter = new ConcurrencyCounter(); + MzMLWriteResult result = MzMLWriter.Write( + MzMLFile.Inspect(input), + Path.Combine(_directory, $"output-{threads}.mzML"), + () => new CountingTransform(counter), + new MzMLWriteOptions { MaxDegreeOfParallelism = threads }); + + Assert.Equal(200, result.SpectraSeen); + Assert.True( + counter.Peak <= threads, + $"asked for {threads} thread(s), but {counter.Peak} spectra were in flight at once"); + } + + private sealed class ConcurrencyCounter + { + private int _current; + private int _peak; + + public int Peak => Volatile.Read(ref _peak); + + public void Enter() + { + int now = Interlocked.Increment(ref _current); + + // Raise the recorded peak to `now` unless something already recorded higher. + int peak = Volatile.Read(ref _peak); + while (now > peak) + { + int seen = Interlocked.CompareExchange(ref _peak, now, peak); + if (seen == peak) break; + peak = seen; + } + } + + public void Exit() => Interlocked.Decrement(ref _current); + } + + /// + /// Holds the worker briefly so overlapping work actually overlaps. Without the pause an + /// unbounded writer could still finish each spectrum before dispatching the next and look + /// bounded by luck. + /// + private sealed class CountingTransform : IMzTransform + { + private readonly ConcurrencyCounter _counter; + + public CountingTransform(ConcurrencyCounter counter) => _counter = counter; + + public MzTransformResult Transform(SpectrumRecord spectrum, Span corrected) + { + _counter.Enter(); + try + { + Thread.Sleep(2); + spectrum.Mz.CopyTo(corrected); + return new MzTransformResult { Rewrite = true }; + } + finally + { + _counter.Exit(); + } + } + } +} diff --git a/dotnet/MARS.Test/PwizOutputTest.cs b/dotnet/MARS.Test/PwizOutputTest.cs new file mode 100644 index 0000000..6c5175e --- /dev/null +++ b/dotnet/MARS.Test/PwizOutputTest.cs @@ -0,0 +1,219 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using System.Linq; +using MARS.Core; +using MARS.IO; +using MARS.Pwiz; +using Xunit; + +namespace MARS.Test; + +public class PwizOutputTest +{ + [Theory] + [InlineData("mzML", MarsOutputFormat.MzML)] + [InlineData("mzml", MarsOutputFormat.MzML)] + [InlineData("MZML", MarsOutputFormat.MzML)] + [InlineData(" mzXML ", MarsOutputFormat.MzXml)] + [InlineData("mzmlb", MarsOutputFormat.MzMLb)] + [InlineData("mgf", MarsOutputFormat.Mgf)] + public void FormatNamesParse(string name, MarsOutputFormat expected) + { + Assert.True(PwizOutput.TryParse(name, out MarsOutputFormat format)); + Assert.Equal(expected, format); + } + + /// An absent --output-format means mzML, the format MARS has always written. + [Theory] + [InlineData(null)] + [InlineData("")] + public void NoFormatMeansMzML(string? name) + { + Assert.True(PwizOutput.TryParse(name, out MarsOutputFormat format)); + Assert.Equal(MarsOutputFormat.MzML, format); + } + + [Theory] + [InlineData("mz5")] + [InlineData("mzdata")] + [InlineData("parquet")] + public void AnUnknownFormatIsRejected(string name) => + Assert.False(PwizOutput.TryParse(name, out _)); + + [Fact] + public void EveryFormatHasAnExtensionAndAName() + { + foreach (MarsOutputFormat format in Enum.GetValues()) + { + Assert.StartsWith(".", PwizOutput.Extension(format), StringComparison.Ordinal); + Assert.True(PwizOutput.TryParse(PwizOutput.Name(format), out MarsOutputFormat parsed)); + Assert.Equal(format, parsed); + } + } + + /// + /// Splicing means copying the input and replacing the ranges that changed, so it needs an + /// mzML to copy. An mzML input can be spliced; a vendor file has nothing to splice into + /// and its mzML has to be built like any other format. + /// + [Theory] + [InlineData("run.mzML", true)] + [InlineData("RUN.MZML", true)] + [InlineData("run.raw", false)] + [InlineData("run.wiff2", false)] + [InlineData("run.d", false)] + public void OnlyAnMzMLInputCanBeSpliced(string path, bool expected) => + Assert.Equal(expected, SpectrumSources.CanSplice(path)); + + /// Every format MARS knows the name of is recognized, so Open can explain itself. + [Theory] + [InlineData("run.mzML")] + [InlineData("run.raw")] + [InlineData("run.wiff2")] + [InlineData("run.lcd")] + public void TheKnownFormatsAreRecognized(string path) => + Assert.True(SpectrumSources.IsRecognized(path)); + + [Theory] + [InlineData("notes.txt")] + [InlineData("run.mzXML")] + public void AnUnknownFormatIsNotRecognized(string path) => + Assert.False(SpectrumSources.IsRecognized(path)); + + /// + /// Recognizing a format is not the same as being able to open it. Shimadzu is in the table + /// so that a .lcd earns an explanation, but no build carries the reader, so a directory + /// scan must not pick one up and a capability list must not advertise it. + /// + [Fact] + public void ARecognizedFormatWithNoReaderIsNotReadable() + { + Assert.True(SpectrumSources.IsRecognized("run.lcd")); + Assert.False(SpectrumSources.IsReadable("run.lcd")); + Assert.DoesNotContain(".lcd", SpectrumSources.ReadableExtensions()); + } + + /// mzML is readable in every build; the vendor formats only where they are linked. + [Fact] + public void TheCapabilityListMatchesWhatIsLinked() + { + Assert.Contains(".mzML", SpectrumSources.ReadableExtensions()); + Assert.Equal(PwizOutput.Available, SpectrumSources.ReadableExtensions().Contains(".raw")); + Assert.Equal(PwizOutput.Available, SpectrumSources.IsReadable("run.raw")); + } + + /// + /// A vendor format asked of a build that cannot open it must say so, rather than failing + /// somewhere inside a reader with a load error. + /// + [Fact] + public void AVendorFormatThisBuildCannotReadIsRefusedClearly() + { + string path = Path.Combine(Path.GetTempPath(), "nonexistent-" + Guid.NewGuid().ToString("N") + ".wiff2"); + + // Sciex is never referenced, so this is refused whether or not pwiz is present. + var ex = Assert.Throws(() => SpectrumSources.Open(path)); + Assert.Contains("Sciex", ex.Message, StringComparison.Ordinal); + } + + /// + /// mzML is always available; the rest depend on whether this build found pwiz-sharp. A + /// build without it must still write mzML, which is what MARS has always done. + /// + [Fact] + public void MzMLIsSupportedWithOrWithoutPwiz() + { + Assert.Contains(MarsOutputFormat.MzML, PwizOutput.Supported); + if (!PwizOutput.Available) Assert.Single(PwizOutput.Supported); + } + + /// + /// mzMLb is HDF5, and the native library that writes it is published for x64 only. It has + /// to be absent from the list on arm64 rather than fail when someone asks for it. + /// + [Fact] + public void MzMLbIsOfferedOnlyWhereItCanBeWritten() + { + bool x64 = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture + == System.Runtime.InteropServices.Architecture.X64; + + Assert.Equal( + PwizOutput.Available && x64, + PwizOutput.Supported.Contains(MarsOutputFormat.MzMLb)); + } + + /// The formats that drop information say so, so a user is warned rather than surprised. + [Fact] + public void TheLossyFormatsCarryAWarning() + { + Assert.Null(PwizOutput.LossWarning(MarsOutputFormat.MzML)); + Assert.Null(PwizOutput.LossWarning(MarsOutputFormat.MzMLb)); + Assert.NotNull(PwizOutput.LossWarning(MarsOutputFormat.MzXml)); + Assert.NotNull(PwizOutput.LossWarning(MarsOutputFormat.Mgf)); + } + + // ---- encoding sniffing ------------------------------------------------------------- + // + // This is the part that has to be right whether or not pwiz is present: pwiz's encoder + // defaults to 64-bit UNCOMPRESSED, and taking that default made a Stellar run 61% larger + // than its input. MARS matches what the input used instead. + + [Theory] + [InlineData(true, true, true, true)] + [InlineData(true, true, false, true)] // 64-bit m/z, 32-bit intensity: the common case + [InlineData(true, false, true, false)] // uncompressed both + [InlineData(false, true, false, true)] + public void TheEncodingIsReadBackPerArray( + bool mzBits64, bool mzZlib, bool intensityBits64, bool intensityZlib) + { + string directory = NewDirectory(); + try + { + string path = Path.Combine(directory, "input.mzML"); + SyntheticMzML.Write( + path, spectrumCount: 8, chromatogramCount: 0, + mzEncoding: new BinaryArrayEncoding(mzBits64, mzZlib), + intensityEncoding: new BinaryArrayEncoding(intensityBits64, intensityZlib), + peaksPerSpectrum: 12); + + SpectrumEncoding encoding = MzMLEncoding.Sniff(path); + + Assert.Equal(mzBits64, encoding.Mz.Bits64); + Assert.Equal(mzZlib, encoding.Mz.Zlib); + Assert.Equal(intensityBits64, encoding.Intensity.Bits64); + Assert.Equal(intensityZlib, encoding.Intensity.Zlib); + } + finally + { + Delete(directory); + } + } + + /// + /// A file this cannot read is not a reason to fail a conversion - fall back to what + /// msconvert writes, which is what the input most likely was. + /// + [Fact] + public void AnUnreadableFileFallsBackToTheUsualEncoding() + { + SpectrumEncoding encoding = MzMLEncoding.Sniff( + Path.Combine(Path.GetTempPath(), "does-not-exist-" + Guid.NewGuid().ToString("N") + ".mzML")); + + Assert.True(encoding.Mz.Bits64); + Assert.True(encoding.Mz.Zlib); + } + + private static string NewDirectory() + { + string directory = Path.Combine(Path.GetTempPath(), "mars-pwiz-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + return directory; + } + + private static void Delete(string directory) + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } +} diff --git a/dotnet/MARS.Test/PwizSpectrumListTest.cs b/dotnet/MARS.Test/PwizSpectrumListTest.cs new file mode 100644 index 0000000..2fc532d --- /dev/null +++ b/dotnet/MARS.Test/PwizSpectrumListTest.cs @@ -0,0 +1,310 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Compiled only when a pwiz-sharp checkout is present; see MARS.Test.csproj. + +using System; +using System.Collections.Generic; +using System.Linq; +using MARS.Core; +using MARS.Pwiz; +using Pwiz.Data.Common.Cv; +using Pwiz.Data.MsData.Spectra; +using Xunit; + +namespace MARS.Test; + +/// +/// The batching in MarsSpectrumList. +/// +/// +/// +/// This is the riskiest code in the pwiz path and the least visible. pwiz's writers pull +/// spectra one at a time, and 79% of a conversion's time is scoring the model, so MARS reads a +/// batch ahead and corrects it in parallel. That means index arithmetic, a reset path for +/// callers that do not walk in order, and a guard against handing the same spectrum out twice - +/// none of which announces itself when it goes wrong. A batching bug would reorder or drop +/// spectra in a written file, which no exception would report. +/// +/// +/// Driven through a fake inner list rather than a real vendor file, so it runs without one and +/// can be pushed into the edge cases a real file would not reach. +/// +/// +public class PwizSpectrumListTest +{ + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(8)] + public void EverySpectrumIsReturnedOnceInOrder(int threads) + { + var inner = new FakeSpectrumList(count: 101); + using var list = Wrap(inner, threads); + + var ids = new List(); + for (int i = 0; i < list.Count; i++) + ids.Add(list.GetSpectrum(i, getBinaryData: true).Id); + + Assert.Equal(Enumerable.Range(0, 101).Select(i => $"scan={i}"), ids); + } + + /// + /// A count that is not a multiple of the batch size leaves a short final batch. Reading off + /// the end of it is the obvious way to get this wrong. + /// + [Theory] + [InlineData(1)] + [InlineData(33)] + [InlineData(64)] + [InlineData(65)] + public void AShortFinalBatchIsServedCompletely(int count) + { + var inner = new FakeSpectrumList(count); + using var list = Wrap(inner, threads: 8); + + for (int i = 0; i < count; i++) + Assert.Equal($"scan={i}", list.GetSpectrum(i, getBinaryData: true).Id); + } + + /// + /// pwiz's writers walk in order, but nothing in the interface promises it. An out-of-order + /// pull has to be served correctly rather than from the wrong place in a read-ahead batch. + /// + [Fact] + public void OutOfOrderAccessIsStillCorrect() + { + var inner = new FakeSpectrumList(count: 60); + using var list = Wrap(inner, threads: 8); + + foreach (int index in new[] { 0, 1, 2, 40, 41, 3, 59, 0, 30 }) + Assert.Equal($"scan={index}", list.GetSpectrum(index, getBinaryData: true).Id); + } + + /// + /// The whole point of the batch is the correction, so it has to actually happen - and only + /// on MS2, which is all MARS calibrates. + /// + [Fact] + public void Ms2IsCorrectedAndMs1IsNot() + { + var inner = new FakeSpectrumList(count: 40); + using var list = Wrap(inner, threads: 8); + + var corrected = new List(); + var untouched = new List(); + for (int i = 0; i < inner.Count; i++) + { + Spectrum s = list.GetSpectrum(i, getBinaryData: true); + double first = s.GetMZArray()!.Data[0]; + if (s.Params.CvParamValueOrDefault(CVID.MS_ms_level, 0) == 2) corrected.Add(first); + else untouched.Add(first); + } + + Assert.NotEmpty(corrected); + Assert.NotEmpty(untouched); + + // MS1 keeps the value the fake produced; MS2 does not. + Assert.All(untouched, v => Assert.Equal(FakeSpectrumList.BaseMz, v, 12)); + Assert.Contains(corrected, v => Math.Abs(v - FakeSpectrumList.BaseMz) > 1e-9); + } + + /// + /// Correcting in parallel must not change the answer. Verified on real data by hashing an + /// mzXML written on 1 thread against one written on 12; this pins it without a 5-minute + /// conversion. + /// + [Fact] + public void TheResultDoesNotDependOnThreadCount() + { + double[] one = CorrectedMz(threads: 1); + double[] many = CorrectedMz(threads: 12); + + Assert.Equal(one.Length, many.Length); + for (int i = 0; i < one.Length; i++) + Assert.Equal(one[i], many[i], 15); + } + + /// Counters have to survive being summed across workers. + [Fact] + public void TheCountersAgreeAcrossThreadCounts() + { + var single = new FakeSpectrumList(count: 101); + using MarsSpectrumList a = Wrap(single, threads: 1); + for (int i = 0; i < single.Count; i++) a.GetSpectrum(i, getBinaryData: true); + + var parallel = new FakeSpectrumList(count: 101); + using MarsSpectrumList b = Wrap(parallel, threads: 12); + for (int i = 0; i < parallel.Count; i++) b.GetSpectrum(i, getBinaryData: true); + + Assert.Equal(a.SpectraSeen, b.SpectraSeen); + Assert.Equal(a.SpectraCorrected, b.SpectraCorrected); + Assert.True(a.SpectraSeen > 0); + } + + /// A metadata-only pull must not prime a batch of fully decoded spectra. + [Fact] + public void MetadataOnlyPullsAreCheap() + { + var inner = new FakeSpectrumList(count: 60); + using var list = Wrap(inner, threads: 8); + + for (int i = 0; i < inner.Count; i++) list.GetSpectrum(i, getBinaryData: false); + + Assert.Equal(0, inner.BinaryReads); + } + + /// + /// Nothing in SpectrumList promises a spectrum is asked for once. A consumer that + /// looks at one twice has to get it twice. + /// + [Fact] + public void ASpectrumCanBeReadTwice() + { + var inner = new FakeSpectrumList(count: 60); + using var list = Wrap(inner, threads: 8); + + Spectrum first = list.GetSpectrum(5, getBinaryData: true); + Spectrum again = list.GetSpectrum(5, getBinaryData: true); + + Assert.Equal("scan=5", first.Id); + Assert.Equal("scan=5", again.Id); + + // And corrected identically both times - the second read must not skip the model. + Assert.Equal(first.GetMZArray()!.Data[0], again.GetMZArray()!.Data[0], 15); + } + + /// + /// A spectrum served outside the batch must add to the file's counters, not replace them. + /// + [Fact] + public void CountersSurviveASpectrumServedOutsideTheBatch() + { + var inner = new FakeSpectrumList(count: 101); + using var list = Wrap(inner, threads: 8); + + for (int i = 0; i < inner.Count; i++) list.GetSpectrum(i, getBinaryData: true); + long seen = list.SpectraSeen; + Assert.True(seen > 1); + + // 101 spectra with every fourth an MS1, and only MS2 is counted. + Assert.Equal(75, seen); + + // A re-read is served outside the batch, and adds exactly one to the totals rather + // than replacing them. Index 99 is an MS2; an MS1 would correctly add nothing. + list.GetSpectrum(99, getBinaryData: true); + Assert.Equal(seen + 1, list.SpectraSeen); + } + + /// + /// A caller that jumps once and then walks in order should get the read-ahead back rather + /// than falling back to one spectrum at a time for the rest of the file. + /// + [Fact] + public void AJumpStartsANewBatchRatherThanDisablingBatching() + { + var inner = new FakeSpectrumList(count: 200); + using var list = Wrap(inner, threads: 8); + + list.GetSpectrum(0, getBinaryData: true); + list.GetSpectrum(120, getBinaryData: true); + + int afterJump = inner.BinaryReads; + + // The batch primed at 120 covers what follows, so walking on reads nothing more. + list.GetSpectrum(121, getBinaryData: true); + list.GetSpectrum(122, getBinaryData: true); + + Assert.Equal(afterJump, inner.BinaryReads); + Assert.Equal("scan=122", list.GetSpectrum(122, getBinaryData: true).Id); + } + + private static double[] CorrectedMz(int threads) + { + var inner = new FakeSpectrumList(count: 101); + using var list = Wrap(inner, threads); + + var values = new List(); + for (int i = 0; i < inner.Count; i++) + { + Spectrum s = list.GetSpectrum(i, getBinaryData: true); + values.AddRange(s.GetMZArray()!.Data); + } + + return values.ToArray(); + } + + private static MarsSpectrumList Wrap(FakeSpectrumList inner, int threads) => + new(inner, TinyModel(), new CorrectionOptions(), acquisitionStart: 0, temperatures: null, threads); + + /// A model that moves m/z measurably, so a missed correction is visible. + private static MzCalibrator TinyModel() + { + var table = new MatchTable(new[] { MarsFeature.FragmentMz, MarsFeature.LogIntensity }); + var random = new Random(3); + + for (var i = 0; i < 400; i++) + { + double fragmentMz = 300 + (random.NextDouble() * 700); + double intensity = 500 + (random.NextDouble() * 100000); + + table.Set(MarsFeature.FragmentMz, fragmentMz); + table.Set(MarsFeature.LogIntensity, Math.Log10(intensity)); + table.DeltaMz.Add(0.01 + (fragmentMz * 1e-5)); + table.ObservedIntensity.Add(intensity); + table.PeptideGroup.Add(i / 8); + table.CommitRow(); + } + + return MzCalibrator.Fit(table, new CalibrationOptions { CvFolds = 0 }, absoluteTimeOffset: 0); + } + + /// + /// A spectrum list that answers from nothing, so the batching can be driven directly. + /// Every fourth spectrum is MS1, as a real DIA run alternates. + /// + private sealed class FakeSpectrumList : SpectrumListBase + { + public const double BaseMz = 500.0; + + public FakeSpectrumList(int count) => Count = count; + + public override int Count { get; } + + /// How many times a caller asked for decoded arrays. + public int BinaryReads { get; private set; } + + public override SpectrumIdentity SpectrumIdentity(int index) => + new() { Index = index, Id = $"scan={index}" }; + + public override Spectrum GetSpectrum(int index, bool getBinaryData = false) + { + if (getBinaryData) BinaryReads++; + + int msLevel = index % 4 == 0 ? 1 : 2; + var spectrum = new Spectrum { Index = index, Id = $"scan={index}" }; + spectrum.Params.Set(CVID.MS_ms_level, msLevel.ToString()); + spectrum.Params.Set(CVID.MS_total_ion_current, "1000000"); + + var scan = new Scan(); + scan.Set(CVID.MS_scan_start_time, (index * 0.01).ToString(), CVID.UO_minute); + scan.Set(CVID.MS_ion_injection_time, (10.0 + (index % 5)).ToString(), CVID.UO_millisecond); + spectrum.ScanList.Scans.Add(scan); + + var precursor = new Precursor(); + precursor.IsolationWindow.Set(CVID.MS_isolation_window_target_m_z, "600"); + precursor.IsolationWindow.Set(CVID.MS_isolation_window_lower_offset, "5"); + precursor.IsolationWindow.Set(CVID.MS_isolation_window_upper_offset, "5"); + spectrum.Precursors.Add(precursor); + + if (!getBinaryData) return spectrum; + + var mz = new BinaryDataArray { Data = { BaseMz, BaseMz + 1, BaseMz + 2 } }; + mz.Params.Set(CVID.MS_m_z_array); + var intensity = new BinaryDataArray { Data = { 5000.0, 6000.0, 7000.0 } }; + intensity.Params.Set(CVID.MS_intensity_array); + spectrum.BinaryDataArrays.Add(mz); + spectrum.BinaryDataArrays.Add(intensity); + spectrum.DefaultArrayLength = 3; + return spectrum; + } + } +} diff --git a/dotnet/MARS.Test/QcReportTest.cs b/dotnet/MARS.Test/QcReportTest.cs new file mode 100644 index 0000000..c4769c7 --- /dev/null +++ b/dotnet/MARS.Test/QcReportTest.cs @@ -0,0 +1,336 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Text.RegularExpressions; +using MARS.Core; +using MARS.Report; +using Xunit; + +namespace MARS.Test; + +public sealed class PngTest +{ + [Fact] + public void ProducesAStructurallyValidPng() + { + const int width = 7, height = 5; + var pixels = new byte[width * height * 3]; + for (int i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i % 251); + + byte[] png = Png.Encode(pixels, width, height); + + Assert.Equal(new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 }, png[..8]); + + var chunks = new List<(string Type, byte[] Data)>(); + int position = 8; + while (position < png.Length) + { + int length = BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(position)); + string type = System.Text.Encoding.ASCII.GetString(png, position + 4, 4); + byte[] data = png[(position + 8)..(position + 8 + length)]; + + // A wrong CRC is the failure mode that produces a file every viewer rejects + // while the bytes look plausible, so check it rather than trusting the writer. + uint recorded = BinaryPrimitives.ReadUInt32BigEndian(png.AsSpan(position + 8 + length)); + Assert.Equal(Crc32(png.AsSpan(position + 4, 4 + length)), recorded); + + chunks.Add((type, data)); + position += 12 + length; + } + + Assert.Equal(png.Length, position); + Assert.Equal(new[] { "IHDR", "IDAT", "IEND" }, chunks.ConvertAll(c => c.Type).ToArray()); + + byte[] header = chunks[0].Data; + Assert.Equal(width, BinaryPrimitives.ReadInt32BigEndian(header.AsSpan(0))); + Assert.Equal(height, BinaryPrimitives.ReadInt32BigEndian(header.AsSpan(4))); + Assert.Equal(8, header[8]); + Assert.Equal(2, header[9]); + + using var input = new MemoryStream(chunks[1].Data); + using var inflate = new ZLibStream(input, CompressionMode.Decompress); + using var output = new MemoryStream(); + inflate.CopyTo(output); + byte[] scanlines = output.ToArray(); + + Assert.Equal(height * ((width * 3) + 1), scanlines.Length); + for (int y = 0; y < height; y++) + { + int offset = y * ((width * 3) + 1); + Assert.Equal(0, scanlines[offset]); // filter type "none" + Assert.Equal( + pixels[(y * width * 3)..((y + 1) * width * 3)], + scanlines[(offset + 1)..(offset + 1 + (width * 3))]); + } + } + + [Fact] + public void RejectsAMismatchedPixelBuffer() + { + Assert.Throws(() => Png.Encode(new byte[10], 4, 4)); + } + + private static uint Crc32(ReadOnlySpan data) + { + uint crc = 0xFFFFFFFF; + foreach (byte b in data) + { + crc ^= b; + for (int i = 0; i < 8; i++) + crc = (crc & 1) != 0 ? 0xEDB88320 ^ (crc >> 1) : crc >> 1; + } + + return crc ^ 0xFFFFFFFF; + } +} + +public sealed class QcHtmlReportTest +{ + private static QcHtmlReport.Data BuildData(int rows = 400) + { + var before = new double[rows]; + var after = new double[rows]; + var rt = new double[rows]; + var mz = new double[rows]; + var feature = new double[rows]; + + for (int i = 0; i < rows; i++) + { + // A deterministic wobble, so the figures have structure to draw rather than a + // flat line, without needing a random source. + double t = i / (double)rows; + before[i] = (0.08 * Math.Sin(t * 6.0)) + (((i * 37) % 19) - 9) * 0.004; + after[i] = before[i] * 0.4; + rt[i] = t * 30.0; + mz[i] = 300 + (t * 900); + feature[i] = Math.Log10(1000 + (i * 17)); + } + + return new QcHtmlReport.Data + { + ErrorBefore = before, + ErrorAfter = after, + RetentionTime = rt, + FragmentMz = mz, + Features = new[] { ("log_intensity", feature) }, + ImportanceNames = new[] { "log_intensity" }, + Importance = new[] { 1.0 }, + }; + } + + private static string WriteReport(QcHtmlReport.Data data) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".html"); + QcHtmlReport.Write( + path, data, statistics: null, + new MARS.Core.MatchStatistics { SpectraSeen = 10, FragmentsMatched = 400, UniqueEntriesMatched = 5 }, + new[] { "run.mzML" }, "0.3 Th", "26.1.0"); + return path; + } + + [Fact] + public void IsSelfContained() + { + string path = WriteReport(BuildData()); + try + { + string html = File.ReadAllText(path); + + // The report exists to be emailed. Anything fetched at open time would render + // as a broken box for the recipient, and most mail clients block it outright. + Assert.DoesNotContain(" new() + { + Rows = 100, Mad = mad, Rms = mad * 2, StdDev = mad * 2, + Median = 0, PearsonR = r, MadBefore = 0.08, + }; + + var withCv = new QcHtmlReport.Data + { + ErrorBefore = data.ErrorBefore, + ErrorAfter = data.ErrorAfter, + RetentionTime = data.RetentionTime, + FragmentMz = data.FragmentMz, + Features = data.Features, + ImportanceNames = data.ImportanceNames, + Importance = data.Importance, + CrossValidation = new CrossValidationReport + { + Folds = 3, + Groups = 60, + PerFold = new[] { Fold(0.044, 0.69), Fold(0.045, 0.68), Fold(0.046, 0.70) }, + OutOfFold = Fold(0.045, 0.69), + InSample = Fold(0.043, 0.71), + }, + }; + + QcHtmlReport.Write( + path, withCv, statistics: null, + new MARS.Core.MatchStatistics { SpectraSeen = 10, FragmentsMatched = 400 }, + new[] { "run.mzML" }, "0.3 Th", "26.1.0"); + + string html = File.ReadAllText(path); + + // The per-fold table, and the spread row that says whether one held-out number + // was luck. A report that showed only the pooled figure would be hiding the + // variance the folds exist to measure. + Assert.Contains("Cross-validation", html, StringComparison.Ordinal); + Assert.Contains("+/-", html, StringComparison.Ordinal); + Assert.Contains("Median absolute residual per fold", html, StringComparison.Ordinal); + Assert.Contains("Pearson correlation per fold", html, StringComparison.Ordinal); + // The gap between what the correction achieves on the data it was fitted to and + // what it would achieve elsewhere. Named "gap" rather than "optimism" because + // calibrating a run from its own species is not cheating - see qc-report.md. + Assert.Contains("Gap ", html, StringComparison.Ordinal); + + // Histogram, the heatmap pair, importance, two fold-spread figures, one per feature. + Assert.Equal(withCv.Features.Count + 5, Regex.Matches(html, "(), + RetentionTime = data.RetentionTime, + FragmentMz = data.FragmentMz, + Features = data.Features, + ImportanceNames = Array.Empty(), + Importance = Array.Empty(), + }; + + QcHtmlReport.Write( + path, preCalibration, statistics: null, + new MARS.Core.MatchStatistics { SpectraSeen = 10, FragmentsMatched = 400 }, + new[] { "run.mzML" }, "0.3 Th", "26.1.0", + MARS.Core.MarsStatistics.Summarize(data.ErrorBefore)); + + string html = File.ReadAllText(path); + + Assert.Contains("Pre-calibration", html, StringComparison.Ordinal); + Assert.Contains("median absolute error", html, StringComparison.Ordinal); + + // Nothing may imply a correction that was never computed. An "after" panel or an + // importance chart here would be reporting a model that does not exist. + Assert.DoesNotContain("After correction", html, StringComparison.Ordinal); + Assert.DoesNotContain("Feature importance", html, StringComparison.Ordinal); + Assert.Contains("As measured", html, StringComparison.Ordinal); + + // Histogram, the heatmap, one panel per feature. + Assert.Equal(preCalibration.Features.Count + 2, Regex.Matches(html, "(), + ErrorAfter = Array.Empty(), + RetentionTime = Array.Empty(), + FragmentMz = Array.Empty(), + Features = Array.Empty<(string, double[])>(), + ImportanceNames = Array.Empty(), + Importance = Array.Empty(), + }; + + string path = WriteReport(empty); + try + { + // A run that matched nothing still has to produce a readable report saying so, + // rather than an exception on the way out. + Assert.Contains("No matched fragments.", File.ReadAllText(path), StringComparison.Ordinal); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void EscapesFileNamesRatherThanInjectingThem() + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".html"); + try + { + // No slashes: Path.GetFileName would truncate at one before escaping ever + // applied, so a name containing "" would not test what it looks like + // it tests. These characters all survive to the escaper. + QcHtmlReport.Write( + path, BuildData(50), statistics: null, + new MARS.Core.MatchStatistics(), + new[] { " & more.mzML" }, "0.3 Th", "26.1.0"); + + string html = File.ReadAllText(path); + Assert.DoesNotContain(" +/// Scan start time carries its own unit, and MARS stores minutes. +/// +/// +/// Thermo records minutes and Bruker records seconds, so reading the value and assuming +/// minutes is wrong by a factor of 60 on half the instruments MARS supports. It went unnoticed +/// until a 64-minute diaPASEF run came back as 3,866 minutes, because nothing here exercised +/// the seconds path - the failure is quiet, feeding the absolute_time feature rather than +/// throwing. +/// +public class ScanTimeUnitTest +{ + [Fact] + public void MinutesAreReadAsMinutes() + { + double[] minutes = RetentionTimes(inSeconds: false); + double[] seconds = RetentionTimes(inSeconds: true); + + Assert.NotEmpty(minutes); + Assert.Equal(minutes.Length, seconds.Length); + } + + /// + /// The same run written in seconds must read back the same minutes. A file that declares + /// seconds and one that declares minutes describe the same acquisition. + /// + [Fact] + public void SecondsAreConvertedToMinutes() + { + double[] minutes = RetentionTimes(inSeconds: false); + double[] seconds = RetentionTimes(inSeconds: true); + + for (int i = 0; i < minutes.Length; i++) + Assert.Equal(minutes[i], seconds[i], 9); + } + + /// + /// Guards the specific mistake: treating seconds as minutes would make these 60x apart. + /// + [Fact] + public void SecondsAreNotTakenAtFaceValue() + { + double[] seconds = RetentionTimes(inSeconds: true); + double[] minutes = RetentionTimes(inSeconds: false); + + Assert.All(seconds, t => Assert.True(t < 60, $"retention time {t} looks like raw seconds")); + Assert.NotEqual(minutes[^1] * 60.0, seconds[^1], 6); + } + + private static double[] RetentionTimes(bool inSeconds) + { + string directory = Path.Combine(Path.GetTempPath(), "mars-rt-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "input.mzML"); + SyntheticMzML.Write( + path, spectrumCount: 24, chromatogramCount: 0, peaksPerSpectrum: 4, + scanTimeInSeconds: inSeconds); + + using var source = new MzMLSpectrumSource(path); + return source.ReadSpectra(msLevel: 2).Select(s => s.RetentionTime).ToArray(); + } + finally + { + try { Directory.Delete(directory, recursive: true); } catch (IOException) { } + } + } +} diff --git a/dotnet/MARS.Test/SpectralLibraryBuilderTest.cs b/dotnet/MARS.Test/SpectralLibraryBuilderTest.cs new file mode 100644 index 0000000..4f8765a --- /dev/null +++ b/dotnet/MARS.Test/SpectralLibraryBuilderTest.cs @@ -0,0 +1,77 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using MARS.Core; +using Xunit; + +namespace MARS.Test; + +/// +/// The builder's per-entry arrays have to stay aligned when an entry is dropped. +/// +/// +/// removes an entry that collected no fragments, +/// which happens routinely - a PRISM row whose product m/z is missing contributes nothing, and +/// a precursor whose rows are all like that ends up empty. Every per-entry array has to shed +/// that entry together, or the arrays index different peptides from each other. +/// +/// The one that matters most is the peptide group. Cross-validation splits folds by peptide +/// specifically so that one peptide's fragments cannot straddle the boundary and let +/// fragment_mz be memorised; a misaligned group array reintroduces exactly that leak, and it +/// does so silently - the reported out-of-fold accuracy simply comes out better than the truth. +/// +public class SpectralLibraryBuilderTest +{ + [Fact] + public void DroppingAnEmptyEntryKeepsEveryArrayAligned() + { + var builder = new SpectralLibraryBuilder(keepSequences: true); + + builder.BeginEntry("PEPTIDEA", 2, 500.0, 1.0, 2.0); + builder.AddFragment(300.0, 100.0, 'y', 3, 1); + builder.EndEntry(); + + // Collects nothing, so EndEntry drops it. + builder.BeginEntry("DROPPEDONE", 2, 600.0, 1.0, 2.0); + builder.EndEntry(); + + builder.BeginEntry("PEPTIDEC", 2, 700.0, 1.0, 2.0); + builder.AddFragment(400.0, 100.0, 'y', 4, 1); + builder.EndEntry(); + + SpectralLibrary library = builder.Build(); + + Assert.Equal(2, library.PrecursorMz.Length); + Assert.Equal(library.PrecursorMz.Length, library.PeptideGroup.Length); + + // The surviving entries must still carry their own groups. With the dropped entry left + // in the group array, entry 1 inherits the group id minted for DROPPEDONE. + Assert.NotEqual(library.PeptideGroup[0], library.PeptideGroup[1]); + Assert.Equal(700.0, library.PrecursorMz[1]); + } + + /// + /// Two entries of the same peptide share a group, which is what keeps them in one fold. + /// A dropped entry in between must not break that. + /// + [Fact] + public void TheSamePeptideKeepsOneGroupAcrossADrop() + { + var builder = new SpectralLibraryBuilder(keepSequences: true); + + builder.BeginEntry("SHARED", 2, 500.0, 1.0, 2.0); + builder.AddFragment(300.0, 100.0, 'y', 3, 1); + builder.EndEntry(); + + builder.BeginEntry("EMPTY", 2, 600.0, 1.0, 2.0); + builder.EndEntry(); + + builder.BeginEntry("SHARED", 3, 350.0, 1.0, 2.0); + builder.AddFragment(310.0, 100.0, 'y', 3, 1); + builder.EndEntry(); + + SpectralLibrary library = builder.Build(); + + Assert.Equal(2, library.PeptideGroup.Length); + Assert.Equal(library.PeptideGroup[0], library.PeptideGroup[1]); + } +} diff --git a/dotnet/MARS.Test/SyntheticMzML.cs b/dotnet/MARS.Test/SyntheticMzML.cs new file mode 100644 index 0000000..1dc5c89 --- /dev/null +++ b/dotnet/MARS.Test/SyntheticMzML.cs @@ -0,0 +1,298 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Builds a small, valid indexed mzML so the test suite needs no data files. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using MARS.IO; + +namespace MARS.Test; + +public static partial class SyntheticMzML +{ + /// + /// Writes an indexed mzML shaped like pwiz output: an indexedmzML wrapper, Thermo + /// nativeIDs, alternating MS1 and MS2 with isolation windows and injection times, and a + /// trailer whose checksum follows the specification. + /// + public static void Write( + string path, + int spectrumCount, + int chromatogramCount, + BinaryArrayEncoding? mzEncoding = null, + BinaryArrayEncoding? intensityEncoding = null, + int seed = 12345, + int peaksPerSpectrum = 0, + MassAnalyzerLayout analyzers = MassAnalyzerLayout.None, + bool constantInjectionTime = false, + bool scanTimeInSeconds = false) + { + BinaryArrayEncoding mzArrayEncoding = mzEncoding ?? new BinaryArrayEncoding(true, true); + BinaryArrayEncoding intensityArrayEncoding = intensityEncoding ?? new BinaryArrayEncoding(true, true); + + var body = new StringBuilder(); + body.Append(""" + + + + + + + + + + + + + + + + + + INSTRUMENT_CONFIGURATION_LIST + + + """.Replace("\r\n", "\n")); + + body.Replace("INSTRUMENT_CONFIGURATION_LIST", InstrumentConfiguration(analyzers)) + .Replace("DEFAULT_CONFIGURATION", DefaultConfigurationAttribute(analyzers)); + + body.Append(" \n"); + + var random = new Random(seed); + for (var i = 0; i < spectrumCount; i++) + { + int scan = i + 1; + int msLevel = i % 4 == 0 ? 1 : 2; + int peaks = peaksPerSpectrum > 0 ? peaksPerSpectrum : 5 + random.Next(40); + + var mz = new double[peaks]; + var intensity = new double[peaks]; + double value = 200.0 + random.NextDouble(); + for (var p = 0; p < peaks; p++) + { + value += 0.5 + (random.NextDouble() * 25.0); + mz[p] = Math.Round(value, 6); + intensity[p] = Math.Round(100.0 + (random.NextDouble() * 50000.0), 4); + } + + double retentionTime = i * 0.01; + double tic = 0; + foreach (double v in intensity) tic += v; + + body.Append(" \n"); + + body.Append(" \n"); + body.Append(" \n"); + body.Append(" \n"); + + body.Append(" \n \n"); + body.Append(" \n" + : "\" unitCvRef=\"UO\" unitAccession=\"UO:0000031\" unitName=\"minute\"/>\n"); + body.Append(" \n"); + body.Append(" \n \n"); + + if (msLevel == 2) + { + double target = 400.0 + (i % 20); + body.Append(" \n \n \n"); + body.Append(" \n"); + body.Append(" \n"); + body.Append(" \n"); + // pwiz writes a userParam that shares the name of a real cvParam; a reader + // matching on name rather than accession trips over exactly this. + body.Append(" \n"); + body.Append(" \n \n \n"); + } + + body.Append(" \n"); + AppendBinaryArray(body, mz, mzArrayEncoding, isMzArray: true); + AppendBinaryArray(body, intensity, intensityArrayEncoding, isMzArray: false); + body.Append(" \n"); + body.Append(" \n"); + } + + body.Append(" \n"); + + if (chromatogramCount > 0) + { + body.Append(" \n"); + + for (var c = 0; c < chromatogramCount; c++) + { + var times = new double[10]; + var values = new double[10]; + for (var p = 0; p < times.Length; p++) + { + times[p] = p * 0.1; + values[p] = 1000.0 * (p + 1); + } + + body.Append(" \n"); + body.Append(" \n"); + body.Append(" \n"); + AppendBinaryArray(body, times, new BinaryArrayEncoding(true, true), isMzArray: false, timeArray: true); + AppendBinaryArray(body, values, new BinaryArrayEncoding(true, true), isMzArray: false); + body.Append(" \n"); + body.Append(" \n"); + } + + body.Append(" \n"); + } + + body.Append(" \n \n"); + + string content = body.ToString(); + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + + // Index offsets are byte positions into the finished stream, so build the index from + // the encoded bytes rather than from character positions. + var spectrumOffsets = new List<(string Id, long Offset)>(); + var chromatogramOffsets = new List<(string Id, long Offset)>(); + CollectOffsets(contentBytes, "\n"); + AppendIndex(index, "spectrum", spectrumOffsets); + AppendIndex(index, "chromatogram", chromatogramOffsets); + index.Append(" \n"); + + byte[] indexBytes = Encoding.UTF8.GetBytes(index.ToString()); + long indexListOffset = contentBytes.Length + 2; // past the two-space indent + byte[] offsetLine = Encoding.UTF8.GetBytes( + " " + indexListOffset.ToString(CultureInfo.InvariantCulture) + "\n"); + byte[] checksumOpen = Encoding.UTF8.GetBytes(" "); + + using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA1); + sha.AppendData(contentBytes); + sha.AppendData(indexBytes); + sha.AppendData(offsetLine); + sha.AppendData(checksumOpen); + string checksum = Convert.ToHexString(sha.GetHashAndReset()).ToLowerInvariant(); + + using FileStream stream = File.Create(path); + stream.Write(contentBytes); + stream.Write(indexBytes); + stream.Write(offsetLine); + stream.Write(checksumOpen); + stream.Write(Encoding.UTF8.GetBytes(checksum + "\n")); + } + + private static void AppendIndex(StringBuilder text, string name, List<(string Id, long Offset)> entries) + { + text.Append(" \n"); + foreach ((string id, long offset) in entries) + { + text.Append(" ") + .Append(offset.ToString(CultureInfo.InvariantCulture)).Append("\n"); + } + + text.Append(" \n"); + } + + private static void CollectOffsets(byte[] content, string openTag, List<(string Id, long Offset)> destination) + { + byte[] needle = Encoding.UTF8.GetBytes(openTag); + var at = 0; + while (true) + { + int found = content.AsSpan(at).IndexOf(needle); + if (found < 0) break; + int absolute = at + found; + + int idAt = content.AsSpan(absolute).IndexOf(" id=\""u8); + if (idAt < 0) break; + int idStart = absolute + idAt + 5; + int idEnd = content.AsSpan(idStart).IndexOf((byte)'"') + idStart; + + destination.Add((Encoding.UTF8.GetString(content, idStart, idEnd - idStart), absolute)); + at = absolute + needle.Length; + } + } + + private static void AppendBinaryArray( + StringBuilder text, double[] values, BinaryArrayEncoding encoding, bool isMzArray, bool timeArray = false) + { + string base64 = EncodeBase64(values, encoding); + + text.Append(" \n"); + text.Append(encoding.Is64Bit + ? " \n" + : " \n"); + text.Append(encoding.Zlib + ? " \n" + : " \n"); + + if (timeArray) + { + text.Append(" \n"); + } + else if (isMzArray) + { + text.Append(" \n"); + } + else + { + text.Append(" \n"); + } + + text.Append(" ").Append(base64).Append("\n"); + text.Append(" \n"); + } + + private static string EncodeBase64(double[] values, BinaryArrayEncoding encoding) + { + byte[] raw; + if (encoding.Is64Bit) + { + raw = new byte[values.Length * 8]; + MemoryMarshal.Cast(values).CopyTo(raw); + } + else + { + var floats = new float[values.Length]; + for (var i = 0; i < values.Length; i++) floats[i] = (float)values[i]; + raw = new byte[values.Length * 4]; + MemoryMarshal.Cast(floats).CopyTo(raw); + } + + if (!encoding.Zlib) return Convert.ToBase64String(raw); + + using var output = new MemoryStream(); + using (var zlib = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + zlib.Write(raw, 0, raw.Length); + } + + return Convert.ToBase64String(output.ToArray()); + } +} diff --git a/dotnet/MARS.Test/SyntheticMzMLAnalyzers.cs b/dotnet/MARS.Test/SyntheticMzMLAnalyzers.cs new file mode 100644 index 0000000..4ec6542 --- /dev/null +++ b/dotnet/MARS.Test/SyntheticMzMLAnalyzers.cs @@ -0,0 +1,93 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Instrument configuration for the synthetic fixture: what says which analyzer measured. + +namespace MARS.Test; + +public static partial class SyntheticMzML +{ + /// Which instrument configuration list the fixture should carry. + public enum MassAnalyzerLayout + { + /// No instrumentConfigurationList at all, as the fixture has always been. + None, + + /// One configuration, a linear ion trap. The shape of a Stellar file. + UnitResolutionTrap, + + /// + /// Two configurations: an orbitrap named as the run default because it takes the MS1 + /// survey, and an Astral analyzer that only the MS2 spectra point at. Classifying this + /// file by its run default gives the wrong answer, which is what makes it worth a + /// fixture - it is the shape of a real Orbitrap Astral file. + /// + HybridOrbitrapAstral, + } + + /// + /// The instrumentConfigurationList element, or nothing. Each case opens with a newline + /// because the placeholder it replaces sits at the end of the preceding line - so the + /// "no configuration" case substitutes to nothing at all and leaves the fixture exactly + /// as it was before this parameter existed. + /// + internal static string InstrumentConfiguration(MassAnalyzerLayout layout) => layout switch + { + MassAnalyzerLayout.UnitResolutionTrap => "\n" + """ + + + + + + + + + + + """, + + MassAnalyzerLayout.HybridOrbitrapAstral => "\n" + """ + + + + + + + + + + + + + + + + + + + + + + + + + """, + + _ => string.Empty, + }; + + /// + /// The run element's defaultInstrumentConfigurationRef attribute. On the hybrid layout this + /// deliberately names IC1, the orbitrap, because that is what a real file does and what + /// makes reading only the run default the wrong way to classify one. + /// + internal static string DefaultConfigurationAttribute(MassAnalyzerLayout layout) => + layout == MassAnalyzerLayout.None ? string.Empty : " defaultInstrumentConfigurationRef=\"IC1\""; + + /// + /// The instrumentConfigurationRef an MS2 spectrum carries. Only the hybrid layout needs + /// one; with a single configuration every spectrum inherits the run default. + /// + internal static string Ms2ConfigurationReference(MassAnalyzerLayout layout, int msLevel) => + layout == MassAnalyzerLayout.HybridOrbitrapAstral && msLevel == 2 + ? " instrumentConfigurationRef=\"IC2\"" + : string.Empty; +} diff --git a/dotnet/MARS.Test/ThreadCountTest.cs b/dotnet/MARS.Test/ThreadCountTest.cs new file mode 100644 index 0000000..5d80d33 --- /dev/null +++ b/dotnet/MARS.Test/ThreadCountTest.cs @@ -0,0 +1,110 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using MARS.Cli; +using Xunit; + +namespace MARS.Test; + +/// +/// How --threads is resolved. +/// +/// +/// The count changes how long a run takes and nothing else - every stage it feeds is +/// per-spectrum or per-feature work with no cross-row accumulation - so a wrong value here is +/// invisible in the output. That is exactly why it is worth pinning: the failure mode is a run +/// that quietly takes four times as long, or one that quietly uses a machine someone else is +/// sharing. +/// +public class ThreadCountTest +{ + [Fact] + public void TheDefaultIsOnePerLogicalProcessor() => + Assert.Equal(Environment.ProcessorCount, Resolve()); + + /// Naming the default explicitly has to mean the same as leaving it out. + [Theory] + [InlineData("auto")] + [InlineData("AUTO")] + [InlineData("Auto")] + public void AutoMeansTheSameAsSayingNothing(string spelling) => + Assert.Equal(Environment.ProcessorCount, Resolve("--threads", spelling)); + + [Fact] + public void ANumberIsTakenAsGiven() => + Assert.Equal(3, Resolve("--threads", "3")); + + /// + /// A count below one is refused rather than read as "use everything". `--threads $N` with + /// N unset expands to nothing or to zero, and quietly taking the whole machine is a poor + /// way to report a scripting mistake. + /// + [Theory] + [InlineData("--threads", "0")] + [InlineData("--threads=-1", null)] + [InlineData("--threads=-16", null)] + public void ACountBelowOneIsRefused(string first, string? second) + { + string[] options = second is null ? new[] { first } : new[] { first, second }; + FormatException error = Assert.Throws(() => Resolve(options)); + Assert.Contains("at least 1", error.Message, StringComparison.Ordinal); + Assert.Contains("auto", error.Message, StringComparison.Ordinal); + } + + /// + /// `--threads -4` is not a negative count to the parser, it is the option followed by + /// another option. Saying so beats reporting that --threads got the value 'true'. + /// + [Fact] + public void AnOptionGivenNoValueSaysSo() + { + FormatException error = Assert.Throws(() => Resolve("--threads", "-4")); + Assert.Contains("no value", error.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("lots")] + [InlineData("8.5")] + [InlineData("")] + public void SomethingThatIsNeitherANumberNorAutoIsRefused(string value) => + Assert.Throws(() => Resolve("--threads", value)); + + /// + /// Asking for more threads than the machine has is allowed - it may be deliberate, and it + /// cannot corrupt anything - but it is said out loud, because it does not go faster. + /// + [Fact] + public void AskingForMoreThreadsThanTheMachineHasWarnsButIsHonoured() + { + var warnings = new List(); + int requested = Environment.ProcessorCount * 4; + + int resolved = ThreadCount.Resolve( + CommandLineArgs.Parse(new[] { "verify", "--threads", requested.ToString() }), + log: null, + warn: warnings.Add); + + Assert.Equal(requested, resolved); + Assert.Single(warnings); + Assert.Contains("logical processors", warnings[0], StringComparison.Ordinal); + } + + /// A run has to be able to say what it settled on; silence is what prompted this. + [Fact] + public void TheChosenCountIsReported() + { + var reported = new List(); + ThreadCount.Resolve(CommandLineArgs.Parse(new[] { "verify" }), reported.Add, warn: null); + + Assert.Single(reported); + Assert.Contains(Environment.ProcessorCount.ToString(), reported[0], StringComparison.Ordinal); + } + + private static int Resolve(params string[] options) + { + var argv = new List { "verify" }; + argv.AddRange(options); + return ThreadCount.Resolve(CommandLineArgs.Parse(argv.ToArray()), log: null, warn: null); + } +} diff --git a/dotnet/MARS.Test/VendoredOspreyTest.cs b/dotnet/MARS.Test/VendoredOspreyTest.cs new file mode 100644 index 0000000..9571e4c --- /dev/null +++ b/dotnet/MARS.Test/VendoredOspreyTest.cs @@ -0,0 +1,196 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Text.Json; +using pwiz.Osprey.ML; +using Xunit; + +namespace MARS.Test; + +/// +/// Drift guard for the vendored Osprey.ML sources. Osprey.ML owns this code; MARS carries a +/// copy only because pwiz has no package feed yet. These tests turn an accidental local edit +/// into a visible failure instead of a silent fork. +/// +public sealed class VendoredOspreyTest +{ + [Fact] + public void VendoredFilesMatchRecordedHashes() + { + string vendorDirectory = TestPaths.VendoredOspreyDirectory; + string manifestPath = Path.Combine(vendorDirectory, "UPSTREAM.json"); + Assert.True(File.Exists(manifestPath), $"Missing drift-guard manifest: {manifestPath}"); + + using JsonDocument manifest = JsonDocument.Parse(File.ReadAllText(manifestPath)); + JsonElement files = manifest.RootElement.GetProperty("files"); + + var checkedAny = false; + foreach (JsonElement file in files.EnumerateArray()) + { + string name = file.GetProperty("vendored").GetString()!; + string expected = file.GetProperty("sha256").GetString()!; + string path = Path.Combine(vendorDirectory, name); + + Assert.True(File.Exists(path), $"Vendored file is missing: {path}"); + + using FileStream stream = File.OpenRead(path); + string actual = Convert.ToHexString(SHA256.HashData(stream)); + + Assert.True( + string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase), + $"Vendored {name} no longer matches its recorded hash.\n" + + $" recorded {expected}\n actual {actual}\n" + + "Do not edit vendored sources. Fix it upstream in pwiz, then run " + + "scripts/sync-osprey-ml.ps1 -Apply."); + + checkedAny = true; + } + + Assert.True(checkedAny, "The manifest listed no files to check."); + } + + /// + /// XorShift64 is vendored as a fragment rather than a whole file, so its guard is the + /// sequence it produces rather than a text hash. Reformatting upstream is harmless; a + /// change to the shift constants would break every seeded result in Osprey and MARS. + /// + [Fact] + public void XorShift64ProducesTheUpstreamSequence() + { + var rng = new XorShift64(42); + var actual = new ulong[8]; + for (var i = 0; i < actual.Length; i++) actual[i] = rng.Next(); + + // Generated by the reference implementation in + // pwiz_tools/Osprey/Osprey.ML/LinearSvmClassifier.cs. + ulong[] expected = Reference(42, actual.Length); + Assert.Equal(expected, actual); + + // A zero seed must not collapse the generator to a fixed point. + var zeroSeeded = new XorShift64(0); + Assert.NotEqual(0UL, zeroSeeded.Next()); + } + + private static ulong[] Reference(ulong seed, int count) + { + ulong state = seed == 0 ? 1UL : seed; + var values = new ulong[count]; + for (var i = 0; i < count; i++) + { + ulong x = state; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + state = x; + values[i] = x; + } + + return values; + } + + /// + /// The regression objective must fit a continuous target and stay reproducible. This is + /// the MARS-side mirror of the Osprey.Test case, so a bad vendor sync fails here too. + /// + [Fact] + public void SquaredErrorObjectiveFitsAndReproduces() + { + var x = new double[200][]; + var y = new double[200]; + for (var i = 0; i < x.Length; i++) + { + x[i] = new[] { i * 0.05, (i % 5) * 1.1, ((i * 7) % 13) * 0.3 }; + y[i] = (0.4 * x[i][0]) - (0.15 * x[i][2]) + 2.0; + } + + var parameters = new GbtParams + { + Objective = GbtObjective.SquaredError, + NTrees = 60, + MaxDepth = 5, + Subsample = 1.0, + ColSample = 1.0, + MaxBins = 128, + Seed = 42, + }; + + var first = GradientBoostedTrees.Train(x, y, parameters); + var second = GradientBoostedTrees.Train(x, y, parameters); + + double sse = 0, baseline = 0, mean = 0; + foreach (double value in y) mean += value; + mean /= y.Length; + + for (var i = 0; i < x.Length; i++) + { + double score = first.ScoreSingle(x[i]); + Assert.Equal(score, second.ScoreSingle(x[i])); + sse += (y[i] - score) * (y[i] - score); + baseline += (y[i] - mean) * (y[i] - mean); + } + + Assert.True(sse < 0.02 * baseline, $"squared error {sse} should be far below the baseline {baseline}"); + } + + [Fact] + public void ModelDataRoundTripsExactly() + { + var x = new double[120][]; + var y = new double[120]; + for (var i = 0; i < x.Length; i++) + { + x[i] = new[] { (i % 11) * 0.4, i * 0.02 }; + y[i] = (0.25 * x[i][0]) - (0.1 * x[i][1]); + } + + var parameters = new GbtParams + { + Objective = GbtObjective.SquaredError, + NTrees = 25, + MaxDepth = 4, + Subsample = 1.0, + ColSample = 1.0, + Seed = 7, + }; + + var model = GradientBoostedTrees.Train(x, y, parameters); + var reloaded = GradientBoostedTrees.FromModelData(model.ToModelData()); + + foreach (double[] row in x) Assert.Equal(model.ScoreSingle(row), reloaded.ScoreSingle(row)); + } +} + +internal static class TestPaths +{ + /// Walks up from the test binary to the dotnet solution directory. + public static string SolutionDirectory + { + get + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + if (File.Exists(Path.Combine(directory.FullName, "MARS.sln"))) return directory.FullName; + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate MARS.sln above the test binary."); + } + } + + public static string VendoredOspreyDirectory => + Path.Combine(SolutionDirectory, "third_party", "Osprey.ML"); + + /// The repository root, one level above the dotnet solution. + public static string RepositoryRoot => Directory.GetParent(SolutionDirectory)!.FullName; + + /// Example data, when it is present. These files are not in source control. + public static bool TryFindExampleFile(string relativePath, out string fullPath) + { + fullPath = Path.Combine(RepositoryRoot, relativePath); + return File.Exists(fullPath); + } +} diff --git a/dotnet/MARS.sln b/dotnet/MARS.sln new file mode 100644 index 0000000..2ae05b7 --- /dev/null +++ b/dotnet/MARS.sln @@ -0,0 +1,104 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MARS.Core", "MARS.Core\MARS.Core.csproj", "{10103AD2-877F-4173-A37E-1886259BA008}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MARS.IO", "MARS.IO\MARS.IO.csproj", "{EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MARS", "MARS\MARS.csproj", "{1EC71A6D-A0C6-4A02-8601-E622138EBEA6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MARS.Test", "MARS.Test\MARS.Test.csproj", "{87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MARS.OspreyML", "MARS.OspreyML\MARS.OspreyML.csproj", "{C7DEDAE3-4A4D-40BD-810C-02854B3686C3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MARS.Pwiz", "MARS.Pwiz\MARS.Pwiz.csproj", "{D2DD350E-5551-4706-8408-2E396330367C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {10103AD2-877F-4173-A37E-1886259BA008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Debug|x64.ActiveCfg = Debug|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Debug|x64.Build.0 = Debug|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Debug|x86.ActiveCfg = Debug|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Debug|x86.Build.0 = Debug|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Release|Any CPU.Build.0 = Release|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Release|x64.ActiveCfg = Release|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Release|x64.Build.0 = Release|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Release|x86.ActiveCfg = Release|Any CPU + {10103AD2-877F-4173-A37E-1886259BA008}.Release|x86.Build.0 = Release|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Debug|x64.ActiveCfg = Debug|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Debug|x64.Build.0 = Debug|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Debug|x86.ActiveCfg = Debug|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Debug|x86.Build.0 = Debug|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Release|Any CPU.Build.0 = Release|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Release|x64.ActiveCfg = Release|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Release|x64.Build.0 = Release|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Release|x86.ActiveCfg = Release|Any CPU + {EA84C3C3-A063-4FAC-BE48-C07DE33E78AF}.Release|x86.Build.0 = Release|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Debug|x64.ActiveCfg = Debug|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Debug|x64.Build.0 = Debug|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Debug|x86.ActiveCfg = Debug|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Debug|x86.Build.0 = Debug|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Release|Any CPU.Build.0 = Release|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Release|x64.ActiveCfg = Release|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Release|x64.Build.0 = Release|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Release|x86.ActiveCfg = Release|Any CPU + {1EC71A6D-A0C6-4A02-8601-E622138EBEA6}.Release|x86.Build.0 = Release|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Debug|Any CPU.Build.0 = Debug|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Debug|x64.ActiveCfg = Debug|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Debug|x64.Build.0 = Debug|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Debug|x86.ActiveCfg = Debug|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Debug|x86.Build.0 = Debug|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Release|Any CPU.ActiveCfg = Release|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Release|Any CPU.Build.0 = Release|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Release|x64.ActiveCfg = Release|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Release|x64.Build.0 = Release|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Release|x86.ActiveCfg = Release|Any CPU + {87346AE9-54D0-4E15-90BE-DAE4FDCC0A58}.Release|x86.Build.0 = Release|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Debug|x64.ActiveCfg = Debug|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Debug|x64.Build.0 = Debug|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Debug|x86.ActiveCfg = Debug|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Debug|x86.Build.0 = Debug|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Release|Any CPU.Build.0 = Release|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Release|x64.ActiveCfg = Release|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Release|x64.Build.0 = Release|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Release|x86.ActiveCfg = Release|Any CPU + {C7DEDAE3-4A4D-40BD-810C-02854B3686C3}.Release|x86.Build.0 = Release|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Debug|x64.ActiveCfg = Debug|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Debug|x64.Build.0 = Debug|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Debug|x86.ActiveCfg = Debug|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Debug|x86.Build.0 = Debug|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Release|Any CPU.Build.0 = Release|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Release|x64.ActiveCfg = Release|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Release|x64.Build.0 = Release|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Release|x86.ActiveCfg = Release|Any CPU + {D2DD350E-5551-4706-8408-2E396330367C}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/dotnet/MARS/ApplyCommand.cs b/dotnet/MARS/ApplyCommand.cs new file mode 100644 index 0000000..8f225b8 --- /dev/null +++ b/dotnet/MARS/ApplyCommand.cs @@ -0,0 +1,180 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from the apply command in mars/cli.py. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using MARS.Core; +using MARS.IO; +using MARS.Pwiz; + +namespace MARS.Cli; + +public static class ApplyCommand +{ + public static int Run(CommandLineArgs args) + { + if (args.Flag("help", "h")) + { + Console.Error.WriteLine(""" + Usage: mars apply --model [options] [ ...] + + Applies a previously trained model to more files, without rematching or + retraining. + + Options: + --model Trained model (required) + --mzml mzML file or glob (repeatable) + --mzml-dir Directory of mzML files + --output-dir Output directory (default .) + --temperature-dir Directory of RFA2-/RFC2- temperature CSVs + --max-isolation-window + Leave wider isolation windows uncorrected + --on-reorder clamp (default), revert, or allow + --python-compat Reproduce the Python inconsistencies + --threads Worker threads (default: auto, one per + logical processor) + --output-format mzML (default), mzXML, mzMLb or mgf + --validate Check the index and checksum of each output + -v, --verbose Verbose output + """); + return Program.ExitSuccess; + } + + Log.Verbose = args.Flag("verbose", "v"); + + string? modelPath = args.String("model"); + if (modelPath is null) + { + Log.Error("--model is required."); + return Program.ExitInputError; + } + + if (!File.Exists(modelPath)) + { + Log.Error($"Model not found: {modelPath}"); + return Program.ExitInputError; + } + + var patterns = new List(args.Strings("mzml", "mzML")); + patterns.AddRange(args.Positional); + List mzmlFiles = CommandLineArgs.ResolveMzMLFiles(patterns, args.String("mzml-dir")); + if (mzmlFiles.Count == 0) + { + Log.Error("No mzML files found. Use --mzml, --mzml-dir, or pass files as arguments."); + return Program.ExitInputError; + } + + string outputDirectory = args.String("output-dir") ?? "."; + Directory.CreateDirectory(outputDirectory); + + var correctionOptions = new CorrectionOptions + { + MaxIsolationWindowWidth = args.Double("max-isolation-window"), + PythonCompatibility = args.Flag("python-compat"), + Monotonicity = args.String("on-reorder")?.ToLowerInvariant() switch + { + null or "clamp" => MonotonicityPolicy.ClampAscending, + "revert" => MonotonicityPolicy.RevertSpectrum, + "allow" => MonotonicityPolicy.Allow, + var other => throw new FormatException($"--on-reorder expects clamp, revert or allow, got '{other}'."), + }, + }; + + string? temperatureDirectory = args.String("temperature-dir"); + bool validate = args.Flag("validate"); + int threads = ThreadCount.Resolve(args, Log.Info, Log.Warn); + + // Resolved before any file is opened, so an unwritable format fails immediately. + MarsOutputFormat outputFormat = CorrectedFileWriter.ResolveFormat(args); + + // Every option this command reads has been read by now, so a typo can be named rather + // than silently ignored. RejectUnknown only knows an option is real because something + // asked for it. + args.RejectUnknown(); + + MzCalibrator calibrator = MarsModelIo.Load(modelPath); + Log.Info($"Loaded model from {modelPath}"); + Log.Info($" {calibrator.Features.Count} features: {string.Join(", ", calibrator.Features.Names())}"); + Log.Info($" acquisition time offset: {calibrator.AbsoluteTimeOffset:F1} s"); + + // A model trained with the RF temperature features expects them at correction time. + // Absent, they are substituted the way training substitutes a missing one - the + // boosting implementation maps a non-finite feature to 0.0 before binning - so the + // run still completes and still corrects. It just does it with two features pinned + // to a value the model saw for no real spectrum, and nothing about the output says + // so, which is why it is worth a line here. + bool wantsRfa2 = calibrator.Features.Contains(MarsFeature.Rfa2Temp); + bool wantsRfc2 = calibrator.Features.Contains(MarsFeature.Rfc2Temp); + bool modelWantsTemperature = wantsRfa2 || wantsRfc2; + + if (modelWantsTemperature && temperatureDirectory is null) + { + Log.Warn( + "This model was trained with the RF temperature features, but no " + + "--temperature-dir was given. They will be treated as missing for every " + + "spectrum. Pass the directory holding the RFA2-/RFC2- CSVs to use them."); + } + + var stopwatch = Stopwatch.StartNew(); + var failures = 0; + + foreach (string file in mzmlFiles) + { + using ISpectrumSource source = SpectrumSources.Open(file); + TemperatureSet? temperatures = temperatureDirectory is null + ? null + : TemperatureCsvReader.Find(file, temperatureDirectory, Log.Debug); + + // Find always returns a set, empty when nothing matched, so the question is which + // logs it actually carries - not whether it returned one. Asked per generator, + // because a directory can hold the RFA2 log for a run and not the RFC2. + if (temperatures is not null) + { + if (wantsRfa2 && temperatures.Rfa2 is null) WarnMissingLog(file, "RFA2"); + if (wantsRfc2 && temperatures.Rfc2 is null) WarnMissingLog(file, "RFC2"); + } + + string outputFile = CorrectedFileWriter.OutputPathFor(file, outputDirectory, outputFormat); + + Log.Info($"Calibrating: {Path.GetFileName(file)} -> {Path.GetFileName(outputFile)}"); + CorrectedFileWriter.Write( + outputFormat, source, outputFile, calibrator, correctionOptions, temperatures, threads); + + if (!validate) continue; + + // The validator checks an mzML index and its SHA-1 footer, neither of which the + // other formats have. Saying so beats silently reporting nothing. + if (outputFormat != MarsOutputFormat.MzML) + { + Log.Info($" --validate checks the mzML index and checksum; skipped for " + + $"{PwizOutput.Name(outputFormat)}"); + continue; + } + + IndexValidationResult validation = MzMLValidator.Validate(outputFile); + if (validation.IsValid) + { + Log.Info($" index and checksum valid ({validation.SpectrumOffsets:N0} spectrum offsets)"); + } + else + { + failures++; + Log.Error($" output validation FAILED for {Path.GetFileName(outputFile)}"); + foreach (string bad in validation.BadOffsets) Log.Error(" " + bad); + if (validation.ChecksumPresent && !validation.ChecksumValid) + Log.Error($" checksum {validation.RecordedChecksum} != {validation.ComputedChecksum}"); + } + } + + Log.Info($"Done in {stopwatch.Elapsed.TotalSeconds:F1} s. Output directory: {outputDirectory}"); + return failures > 0 ? Program.ExitOutputValidationFailure : Program.ExitSuccess; + } + + private static void WarnMissingLog(string file, string generator) => + Log.Warn( + $"No {generator} temperature log matched {Path.GetFileName(file)}, but the model uses " + + $"{generator.ToLowerInvariant()}_temp. That feature will be treated as missing for " + + "every spectrum in this run."); +} diff --git a/dotnet/MARS/CalibrateCommand.cs b/dotnet/MARS/CalibrateCommand.cs new file mode 100644 index 0000000..aed4874 --- /dev/null +++ b/dotnet/MARS/CalibrateCommand.cs @@ -0,0 +1,607 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from the calibrate command in mars/cli.py. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using MARS.Core; +using MARS.IO; +using MARS.Pwiz; +using MARS.Report; + +namespace MARS.Cli; + +public static class CalibrateCommand +{ + public static int Run(CommandLineArgs args) + { + if (args.Flag("help", "h")) + { + PrintHelp(); + return Program.ExitSuccess; + } + + Log.Verbose = args.Flag("verbose", "v"); + + var patterns = new List(args.Strings("mzml", "mzML")); + patterns.AddRange(args.Positional); + List mzmlFiles = CommandLineArgs.ResolveMzMLFiles(patterns, args.String("mzml-dir")); + + if (mzmlFiles.Count == 0) + { + Log.Error("No mzML files found. Use --mzml, --mzml-dir, or pass files as arguments."); + return Program.ExitInputError; + } + + string outputDirectory = args.String("output-dir") ?? "."; + Directory.CreateDirectory(outputDirectory); + + string modelPath = args.String("model-path") ?? Path.Combine(outputDirectory, "mars_model.json"); + string reportPath = args.String("report") ?? Path.Combine(outputDirectory, "mars_qc_summary.txt"); + string? dumpMatchesPath = args.String("dump-matches"); + string? dumpPredictionsPath = args.String("dump-predictions"); + bool noHtmlReport = args.Flag("no-html-report"); + string htmlReportPath = args.String("html-report") + ?? Path.Combine(outputDirectory, "mars_qc_report.html"); + bool keepDetail = dumpMatchesPath is not null || dumpPredictionsPath is not null || !noHtmlReport; + + // Resolved once and used for both the training histograms and the write, so the run + // reports a single number rather than settling on one per stage. + int threads = ThreadCount.Resolve(args, Log.Info, Log.Warn); + + var matchOptions = new MatchOptions + { + MzToleranceTh = args.Double("tolerance") ?? ResolutionMode.DefaultToleranceTh, + TolerancePpm = args.Double("tolerance-ppm") ?? 0, + MinIntensity = args.Double("min-intensity") ?? 500.0, + MaxIsolationWindowWidth = args.Double("max-isolation-window"), + }; + + var calibrationOptions = new CalibrationOptions + { + NEstimators = args.Int("n-estimators") ?? 100, + MaxDepth = args.Int("max-depth") ?? 6, + LearningRate = args.Double("learning-rate") ?? 0.1, + Seed = args.Int("seed") ?? 42, + ValidationSplit = args.Double("validation-split") ?? 0.2, + CvFolds = args.Int("cv-folds") ?? 5, + Robust = ParseRobust(args.String("robust")), + RobustSigma = args.Double("robust-sigma") ?? 3.0, + MaxTrainingRows = args.Int("max-training-rows") ?? 0, + MaxDegreeOfParallelism = threads, + }; + + var correctionOptions = new CorrectionOptions + { + MaxIsolationWindowWidth = matchOptions.MaxIsolationWindowWidth, + PythonCompatibility = args.Flag("python-compat"), + Monotonicity = ParseMonotonicity(args.String("on-reorder")), + }; + + int minTrainingRows = args.Int("min-training-rows") ?? 1000; + bool noRecalibrate = args.Flag("no-recalibrate"); + string? temperatureDirectory = args.String("temperature-dir"); + + + + // Resolved before any work: an output format this build cannot write, or one that + // does not exist, should cost a second rather than a full training run. + MarsOutputFormat outputFormat = CorrectedFileWriter.ResolveFormat(args); + + // Read before the check below rather than inside LoadLibrary, so that every option + // this command understands has been seen by the time the check runs. + var librarySource = LibrarySource.From(args); + + // Everything is read; refuse a typo now rather than after minutes of work, or worse, + // after writing corrected files from a run that silently used a default. + // Read here, used further down once the readers are open and can say what analyzer + // they saw. RejectUnknown only knows an option is real because something asked for it, + // so an option resolved later has to be touched before the check or it is reported as + // a typo. + ResolutionMode.Touch(args); + + args.RejectUnknown(); + + Log.Info($"Found {mzmlFiles.Count} mzML file(s) to process"); + var stopwatch = Stopwatch.StartNew(); + + // ---- Library ---------------------------------------------------------------- + var runNames = new List(); + foreach (string file in mzmlFiles) runNames.Add(Path.GetFileName(file)); + + SpectralLibrary library = librarySource.Load(runNames, keepSequences: keepDetail, Log.Info); + + // ---- Pass 1: match fragments across every input file ------------------------- + var temperatureByFile = new Dictionary(StringComparer.OrdinalIgnoreCase); + + // Opened once and held: a vendor reader keeps a handle on the file, and reopening it + // per pass would pay the SDK's startup cost twice. + var sourceByFile = new Dictionary(StringComparer.OrdinalIgnoreCase); + + bool anyTemperature = false; + bool anyRfa2 = false, anyRfc2 = false; + foreach (string file in mzmlFiles) + { + sourceByFile[file] = SpectrumSources.Open(file); + + if (temperatureDirectory is not null) + { + TemperatureSet temperatures = TemperatureCsvReader.Find(file, temperatureDirectory, Log.Info); + temperatureByFile[file] = temperatures; + anyTemperature |= !temperatures.IsEmpty; + anyRfa2 |= temperatures.Rfa2 is not null; + anyRfc2 |= temperatures.Rfc2 is not null; + } + } + + // Probed on every file, not just the first. One run in a cohort can record a varying + // injection time where another does not, and the feature group is worth having if any + // of them carries the information - a run without it contributes a constant column + // for its own rows, which costs nothing. + InjectionTimeUse injectionTimeUse = InjectionTimeUse.Absent; + foreach (string file in mzmlFiles) + { + InjectionTimeUse use = ProbeInjectionTime(sourceByFile[file]); + + // One run recording a varying time settles it; there is nothing a later file can + // say that would turn the feature group back off. + if (use == InjectionTimeUse.Varying) + { + injectionTimeUse = use; + break; + } + + if (injectionTimeUse == InjectionTimeUse.Absent) injectionTimeUse = use; + } + + ReportInjectionTime(injectionTimeUse); + + // Decided once the readers are open, from what they say their MS2 analyzer is. The + // readers know their own formats; asking the file again from here would mean parsing + // a .raw as if it were mzML, which is how this used to fall back to a trap tolerance + // on Astral data without anyone noticing. + MassAnalyzerClass analyzer = sourceByFile[mzmlFiles[0]].Analyzer; + + if (FirstAnalyzerDisagreement(mzmlFiles, f => sourceByFile[f].Analyzer, analyzer) is string odd) + { + Log.Warn( + $"{Path.GetFileName(odd)} was recorded on a {Describe(sourceByFile[odd].Analyzer)} " + + $"analyzer, but the fragment tolerance is being set from a {Describe(analyzer)} one. " + + "One tolerance is used for the whole cohort; calibrate the instruments separately, " + + "or set --resolution to choose deliberately."); + } + + ResolutionMode resolution = ResolutionMode.Resolve(args, analyzer, matchOptions, Log.Info); + + MarsFeature[] collect = FragmentMatcher.CollectedFeatures(injectionTimeUse, anyRfa2, anyRfc2); + var table = new MatchTable(collect, keepDetail: keepDetail); + var matcher = new FragmentMatcher(library, matchOptions); + + foreach (string file in mzmlFiles) + { + Log.Info($"Matching: {Path.GetFileName(file)}"); + ISpectrumSource source = sourceByFile[file]; + temperatureByFile.TryGetValue(file, out TemperatureSet? temperatures); + + long before = table.Count; + long spectra = 0; + foreach (SpectrumRecord spectrum in source.ReadSpectra(msLevel: 2)) + { + matcher.MatchSpectrum(spectrum, temperatures, table); + spectra++; + } + + Log.Info($" {spectra:N0} MS2 spectra, {table.Count - before:N0} fragment matches"); + } + + Log.Info($"Total matches: {table.Count:N0} from {matcher.Statistics.SpectraSeen:N0} spectra"); + CheckTolerance(table, matchOptions); + Log.Info($" unique library precursors matched: {matcher.Statistics.UniqueEntriesMatched:N0} " + + $"of {library.EntryCount:N0}"); + + if (table.Count < minTrainingRows) + { + throw new InsufficientTrainingDataException( + $"Only {table.Count:N0} fragment matches; at least {minTrainingRows:N0} are required. " + + "Check that the library and the mzML files describe the same runs, and that the " + + "tolerance is wide enough for the instrument."); + } + + // Re-base acquisition time to the earliest matched spectrum, so the feature starts + // near zero. The offset travels with the model and is subtracted again at + // correction time; feeding raw Unix timestamps to a model trained on re-based ones + // would push every inference row past the largest value it ever saw. + double absoluteTimeOffset = 0; + if (table.Has(MarsFeature.AbsoluteTime)) + { + absoluteTimeOffset = table.MinOf(MarsFeature.AbsoluteTime); + if (double.IsFinite(absoluteTimeOffset)) + { + table.OffsetColumn(MarsFeature.AbsoluteTime, -absoluteTimeOffset); + double span = table.MaxOf(MarsFeature.AbsoluteTime); + Log.Info($"Acquisition time span: 0 to {span:F1} s ({span / 60:F1} min)"); + } + else + { + absoluteTimeOffset = 0; + } + } + + if (dumpMatchesPath is not null) + { + MatchDumpWriter.Write(dumpMatchesPath, table, library); + Log.Info($"Wrote {table.Count:N0} matches to {dumpMatchesPath}"); + } + + // ---- Train ------------------------------------------------------------------- + Log.Info("Training calibration model..."); + MzCalibrator calibrator = MzCalibrator.Fit(table, calibrationOptions, absoluteTimeOffset, Log.Info); + TrainingStatistics stats = calibrator.Statistics!; + + Log.Info($" train MAE {stats.TrainMae:F4} Th, RMSE {stats.TrainRmse:F4} Th"); + if (stats.RowsValidation > 0) + Log.Info($" val MAE {stats.ValidationMae:F4} Th, RMSE {stats.ValidationRmse:F4} Th"); + Log.Info($" delta m/z std {stats.Before.StdDev:F4} -> {stats.After.StdDev:F4} Th " + + $"({PercentReduction(stats.Before.StdDev, stats.After.StdDev):F1}% reduction)"); + Log.Info($" delta m/z MAD {stats.Before.Mad:F4} -> {stats.After.Mad:F4} Th " + + $"({PercentReduction(stats.Before.Mad, stats.After.Mad):F1}% reduction)"); + + if (dumpPredictionsPath is not null) + { + double[] predictions = calibrator.PredictAll(table); + MatchDumpWriter.Write(dumpPredictionsPath, table, library, predictions); + Log.Info($"Wrote {table.Count:N0} predictions to {dumpPredictionsPath}"); + } + + MarsModelIo.Save(calibrator, modelPath); + Log.Info($"Saved model to {modelPath}"); + + QcReport.Write(reportPath, calibrator, matcher.Statistics, mzmlFiles, matchOptions); + Log.Info($"Wrote QC report to {reportPath}"); + + if (!noHtmlReport) + { + QcHtmlReport.Write( + htmlReportPath, + BuildReportData(table, calibrator), + stats, + matcher.Statistics, + mzmlFiles, + DescribeTolerance(matchOptions), + MarsInfo.Version, + uncorrected: null, + resolution.ReportInPpm ? ErrorScale.Ppm : ErrorScale.Th); + Log.Info($"Wrote QC figures to {htmlReportPath}"); + } + + // ---- Pass 2: write corrected files ------------------------------------------- + if (!noRecalibrate) + { + foreach (string file in mzmlFiles) + { + temperatureByFile.TryGetValue(file, out TemperatureSet? temperatures); + CorrectedFileWriter.Write( + outputFormat, + sourceByFile[file], + CorrectedFileWriter.OutputPathFor(file, outputDirectory, outputFormat), + calibrator, + correctionOptions, + temperatures, + threads); + } + } + + foreach (ISpectrumSource source in sourceByFile.Values) source.Dispose(); + + Log.Info($"Done in {stopwatch.Elapsed.TotalSeconds:F1} s. Output directory: {outputDirectory}"); + return Program.ExitSuccess; + } + + + /// + /// Checks the matched error against the window it was matched in, once there is data to + /// check with. Detection can come up empty - a vendor model pwiz does not recognise leaves + /// no analyzer term at all - and this catches the consequence rather than the cause. + /// + internal static void CheckTolerance(MatchTable table, MatchOptions options) + { + if (table.Count == 0) return; + + ReadOnlySpan delta = table.DeltaMz.Items.AsSpan(0, table.Count); + double mad = MarsStatistics.Summarize(delta).Mad; + + double[] fragmentMz = table.Column(MarsFeature.FragmentMz).Items; + var sample = new double[table.Count]; + Array.Copy(fragmentMz, sample, table.Count); + Array.Sort(sample); + double median = sample[sample.Length / 2]; + + ResolutionMode.WarnIfToleranceLooksTooWide(options, mad, median, Log.Warn); + } + /// + /// Whether ion injection time is worth using as a feature. + /// + /// + /// + /// Presence is not enough - it also has to vary. A trap sets injection time per spectrum + /// from its automatic gain control, so it carries real information about how full the trap + /// was. A Bruker or Sciex TOF accumulates for a fixed time, so the value is the same on + /// every spectrum: injection_time is then a constant, which a tree can never split + /// on, and tic_injection_time is TIC times that constant, which is + /// log_tic rescaled. Two features carrying nothing, one of them a duplicate that + /// splits permutation importance with the feature it duplicates. + /// + /// + /// Sampled over the head of the run, which is enough to answer whether the run records an + /// injection time at all - a format that carries none carries none anywhere. It is not + /// enough to answer whether the value varies, and is no longer used for that: see + /// . + /// + /// + /// + /// The first file in a cohort not recorded on the same kind of analyzer as the rest, or + /// null when they agree. + /// + /// + /// One tolerance is chosen for the whole cohort, so a folder holding both trap and + /// high-resolution runs gets one of them matched at the wrong width. That is the quiet + /// failure: matching Astral data at a trap tolerance opens a window hundreds of ppm wide, + /// fills it with wrong assignments, and reports a full model trained on them. It warrants + /// a warning rather than a refusal, because --resolution can be set deliberately and a + /// mixed cohort is the user's call to make. + /// + /// A file whose analyzer could not be read is not a disagreement. It says nothing, and + /// nothing is not a contradiction - it already falls back to the default tolerance. + /// + internal static string? FirstAnalyzerDisagreement( + IReadOnlyList files, + Func analyzerOf, + MassAnalyzerClass chosen) + { + if (chosen == MassAnalyzerClass.Unknown) return null; + + foreach (string file in files) + { + MassAnalyzerClass other = analyzerOf(file); + if (other != chosen && other != MassAnalyzerClass.Unknown) return file; + } + + return null; + } + + private static string Describe(MassAnalyzerClass analyzer) => analyzer switch + { + MassAnalyzerClass.HighResolution => "high-resolution", + MassAnalyzerClass.UnitResolution => "unit-resolution", + _ => "unrecognized", + }; + + internal static InjectionTimeUse ProbeInjectionTime(ISpectrumSource source) + { + const int sample = 500; + + // Loose enough to absorb float representation, orders of magnitude tighter than any + // real gain control. A trap's injection times differ by whole milliseconds. + const double constantWithin = 1e-6; + + int seen = 0; + int withValue = 0; + double low = double.MaxValue; + double high = double.MinValue; + + foreach (SpectrumRecord spectrum in source.ReadSpectra(msLevel: 2)) + { + if (spectrum.InjectionTime is double injection) + { + withValue++; + low = Math.Min(low, injection); + high = Math.Max(high, injection); + } + + if (++seen >= sample) break; + } + + if (withValue == 0) return InjectionTimeUse.Absent; + + double scale = Math.Abs(high) > 0 ? Math.Abs(high) : 1.0; + return (high - low) / scale > constantWithin + ? InjectionTimeUse.Varying + : InjectionTimeUse.Constant; + } + + /// + /// Reports what the probe found. Only the absent case is acted on here. + /// + /// + /// Whether the injection time varies is not decided from this. The probe reads the + /// head of the run, and an ion trap holds its injection time at the method's ceiling until + /// the trap fills - which on a gradient is the whole void volume, tens of thousands of + /// spectra. Every Stellar run tested reads as constant over its first few hundred MS2 and + /// varies later, one of them across two thirds of its spectra. That call belongs where the + /// whole column is available, which is + /// 's feature selection, and it is made there. + /// + internal static InjectionTimeUse ReportInjectionTime(InjectionTimeUse use) + { + if (use == InjectionTimeUse.Absent) + { + Log.Warn("No ion injection time in this run; the ion-population features are off. " + + "They count the ions in a window, and without an injection time there " + + "is nothing to turn a rate into a count with."); + } + + return use; + } + + private static RobustFit ParseRobust(string? value) => value?.ToLowerInvariant() switch + { + null or "trim" => RobustFit.Trim, + "huber" => RobustFit.Huber, + "none" => RobustFit.None, + _ => throw new FormatException($"--robust expects huber, trim or none, got '{value}'."), + }; + + private static MonotonicityPolicy ParseMonotonicity(string? value) => value?.ToLowerInvariant() switch + { + null or "clamp" => MonotonicityPolicy.ClampAscending, + "revert" => MonotonicityPolicy.RevertSpectrum, + "allow" => MonotonicityPolicy.Allow, + _ => throw new FormatException($"--on-reorder expects clamp, revert or allow, got '{value}'."), + }; + + internal static double PercentReduction(double before, double after) => + before > 0 ? (before - after) / before * 100.0 : 0.0; + + /// + /// Collects the per-row values the figures are drawn from. + /// + /// + /// The arrays are handed over rather than copied: the match table's backing store is + /// already column-major in exactly the layout the charts want, and a cohort can carry + /// millions of rows. can be longer than the row + /// count, so every span is bounded by table.Count. + /// + private static QcHtmlReport.Data BuildReportData(MatchTable table, MzCalibrator calibrator) + { + int rows = table.Count; + double[] before = table.DeltaMz.Items[..rows]; + + double[] predictions = calibrator.PredictAll(table); + var after = new double[rows]; + for (int i = 0; i < rows; i++) after[i] = before[i] - predictions[i]; + + var features = new List<(string Name, double[] Values)>(calibrator.Features.Count); + foreach (MarsFeature feature in calibrator.Features.Features) + features.Add((MarsFeatures.NameOf(feature), table.Column(feature).Items[..rows])); + + var importanceNames = new List(calibrator.Features.Count); + foreach (MarsFeature feature in calibrator.Features.Features) + importanceNames.Add(MarsFeatures.NameOf(feature)); + + return new QcHtmlReport.Data + { + ErrorBefore = before, + ErrorAfter = after, + RetentionTime = table.RetentionTime is null ? Array.Empty() : table.RetentionTime.Items[..rows], + FragmentMz = table.Has(MarsFeature.FragmentMz) + ? table.Column(MarsFeature.FragmentMz).Items[..rows] + : Array.Empty(), + Features = features, + ImportanceNames = importanceNames, + Importance = calibrator.Statistics?.PermutationImportance ?? Array.Empty(), + CrossValidation = calibrator.CrossValidation, + }; + } + + private static string DescribeTolerance(MatchOptions options) => + options.TolerancePpm > 0 + ? $"{options.TolerancePpm:0.##} ppm" + : $"{options.MzToleranceTh:0.###} Th"; + + private static void PrintHelp() + { + Console.Error.WriteLine(""" + Usage: mars calibrate [options] [ ...] + + Learns an m/z calibration from spectral library matches and writes recalibrated + mzML files named {input}-mars.mzML. + + Input: + --mzml mzML file or glob (repeatable) + --mzml-dir Directory of mzML files + --prism-csv Skyline PRISM report CSV (theoretical Product Mz) + --library .blib, DIA-NN report-lib.parquet, or PRISM .csv + --diann-report DIA-NN report.parquet, for per-run RT windows + --temperature-dir Directory of RFA2-/RFC2- temperature CSVs + + Matching: + --output-format mzML (default), mzXML, mzMLb or mgf. mzML is + written by splicing the input; the rest are + built through pwiz + --resolution unit, hram or auto (default auto: read the mass + analyzer from the mzML and pick the tolerance + and the QC report's units to match) + --tolerance Fragment tolerance in Th (default 0.3) + --tolerance-ppm Fragment tolerance in ppm; overrides --tolerance + --min-intensity Minimum peak intensity to match (default 500) + --max-isolation-window + Skip spectra with wider isolation windows + --rt-window RT half-window for blib entries (default 0.083) + --no-dedupe-library Keep transitions repeated across replicates + + Model: + --n-estimators Boosting rounds (default 100) + --max-depth Tree depth (default 6) + --learning-rate Shrinkage (default 0.1) + --robust Second pass over rows the first could not explain, + usually mismatched peaks whose delta is not a mass + error at all: trim (default) drops them, huber + holds them down in proportion to how implausible + they are, none fits once + --robust-sigma Residual threshold for --robust, in robust sigma + (default 3). 0 disables the second pass + --cv-folds Cross-validation folds, split by peptide + (default 5). Does not change what gets applied: + the correction model is fitted to all rows either + way. The folds estimate what the same correction + would achieve on data it was not fitted to, which + is reported alongside. 0 skips them + --validation-split Held-out fraction for --cv-folds 0 (default 0.2) + --max-training-rows + Cap training rows by even stride (default no cap) + --min-training-rows + Refuse to fit below this many matches (default 1000) + --seed Random seed (default 42) + + Output: + --output-dir Output directory (default .) + --model-path Where to save the model + --report Where to write the QC summary + --html-report Where to write the QC figures (default + mars_qc_report.html in the output directory). + One self-contained file, safe to email + --no-html-report Skip the figures and write only the text summary + --dump-matches Write every matched fragment to CSV, one row per + match, with all computed features. Diagnostic; + a large cohort produces millions of rows + --dump-predictions + As --dump-matches, plus the model's predicted + correction and the residual, written after + training + --no-recalibrate Train and report only; write no mzML + --on-reorder clamp (default), revert, or allow, when a + correction would break ascending m/z order + --python-compat Reproduce two known inconsistencies in the Python + implementation, for A/B comparison + --threads Worker threads (default: auto, one per + logical processor) + -v, --verbose Verbose output + """); + } +} + +/// Adapts the calibrator to the writer's per-worker transform contract. +internal sealed class CalibratingTransform : IMzTransform +{ + private readonly SpectrumCorrector _corrector; + private readonly TemperatureSet? _temperatures; + private readonly CorrectionWorkspace _workspace = new(); + + public CalibratingTransform(MzCalibrator calibrator, CorrectionOptions options, TemperatureSet? temperatures) + { + _corrector = new SpectrumCorrector(calibrator, options); + _temperatures = temperatures; + } + + public MzTransformResult Transform(SpectrumRecord spectrum, Span corrected) + { + SpectrumCorrectionResult result = _corrector.Correct(spectrum, _temperatures, _workspace, corrected); + return new MzTransformResult + { + Rewrite = result.Corrected, + MonotonicityFixes = result.MonotonicityFixes, + Reverted = result.Reverted, + }; + } +} diff --git a/dotnet/MARS/CommandLineArgs.cs b/dotnet/MARS/CommandLineArgs.cs new file mode 100644 index 0000000..13aa8c9 --- /dev/null +++ b/dotnet/MARS/CommandLineArgs.cs @@ -0,0 +1,292 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace MARS.Cli; + +/// +/// Small hand-rolled option parser. MARS deliberately has no command-line package +/// dependency: the assembly is meant to drop into a managed ProteoWizard tree, and every +/// package it drags along is one more thing that has to be vetted there. +/// +public sealed class CommandLineArgs +{ + private readonly Dictionary> _options = new(StringComparer.OrdinalIgnoreCase); + private readonly List _positional = new(); + // Every name any command has asked about, whether it was supplied or not. This is the + // set of options the running command understands, and it maintains itself: a new option + // is recognized by the act of reading it, so there is no second list to keep in sync. + private readonly HashSet _queried = new(StringComparer.OrdinalIgnoreCase); + + private CommandLineArgs() + { + } + + public string Command { get; private set; } = string.Empty; + + public IReadOnlyList Positional => _positional; + + public static CommandLineArgs Parse(string[] args) + { + var parsed = new CommandLineArgs(); + int start = 0; + + if (args.Length > 0 && !args[0].StartsWith("-", StringComparison.Ordinal)) + { + parsed.Command = args[0]; + start = 1; + } + + for (int i = start; i < args.Length; i++) + { + string arg = args[i]; + if (!arg.StartsWith("--", StringComparison.Ordinal) && !arg.StartsWith("-", StringComparison.Ordinal)) + { + parsed._positional.Add(arg); + continue; + } + + string name = arg.TrimStart('-'); + string? value = null; + + int equals = name.IndexOf('='); + if (equals >= 0) + { + value = name[(equals + 1)..]; + name = name[..equals]; + } + else if (i + 1 < args.Length && !args[i + 1].StartsWith("-", StringComparison.Ordinal)) + { + value = args[++i]; + } + + if (!parsed._options.TryGetValue(name, out List? values)) + { + values = new List(); + parsed._options[name] = values; + } + + values.Add(value ?? "true"); + } + + return parsed; + } + + public bool Has(params string[] names) + { + Query(names); + foreach (string name in names) + { + if (_options.ContainsKey(name)) return true; + } + + return false; + } + + public bool Flag(params string[] names) + { + Query(names); + foreach (string name in names) + { + if (_options.TryGetValue(name, out List? values)) + { + return values.Count == 0 || + !string.Equals(values[^1], "false", StringComparison.OrdinalIgnoreCase); + } + } + + return false; + } + + public string? String(params string[] names) + { + Query(names); + foreach (string name in names) + { + if (_options.TryGetValue(name, out List? values) && values.Count > 0) + return values[^1]; + } + + return null; + } + + public IReadOnlyList Strings(params string[] names) + { + Query(names); + var all = new List(); + foreach (string name in names) + { + if (_options.TryGetValue(name, out List? values)) all.AddRange(values); + } + + return all; + } + + public double? Double(params string[] names) + { + string? text = String(names); + if (text is null) return null; + if (!double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out double value)) + throw new FormatException($"Option --{names[0]} expects a number, got '{text}'."); + return value; + } + + public int? Int(params string[] names) + { + string? text = String(names); + if (text is null) return null; + if (!int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out int value)) + throw new FormatException($"Option --{names[0]} expects an integer, got '{text}'."); + return value; + } + + private void Query(string[] names) + { + foreach (string name in names) _queried.Add(name); + } + + /// Options the command does not understand; almost always a typo. + public IReadOnlyList UnknownOptions() => + _options.Keys.Where(k => !_queried.Contains(k)).OrderBy(k => k, StringComparer.Ordinal).ToList(); + + /// + /// Fails the run if any supplied option is one this command does not understand. + /// + /// + /// + /// Called once each command has read its options and before it starts work, so a typo + /// costs a second rather than the length of a run. It has to be a refusal rather than a + /// warning: a mistyped --tolerance-ppm does not stop MARS, it silently calibrates against + /// the 0.3 Th default, and on a high-resolution instrument that is a wide enough window + /// to admit mostly wrong matches. The run finishes, writes corrected files and reports + /// plausible numbers, all of them meaningless. + /// + /// + /// A misplaced call would be worse than none - an option read after it has not been + /// queried yet and would be rejected while valid - so + /// EveryDocumentedOptionSurvivesTheUnknownOptionCheck passes each command its full + /// documented option set and asserts nothing is rejected. + /// + /// + public void RejectUnknown() + { + IReadOnlyList unknown = UnknownOptions(); + if (unknown.Count == 0) return; + + var message = new System.Text.StringBuilder(); + foreach (string name in unknown) + { + if (message.Length > 0) message.Append("; "); + message.Append($"Unknown option --{name}"); + if (Closest(name) is string suggestion) message.Append($". Did you mean --{suggestion}?"); + } + + message.Append($" (mars {Command} --help lists the options)"); + throw new UnknownOptionException(message.ToString()); + } + + /// The nearest option this command does understand, if one is near enough. + private string? Closest(string name) + { + string? best = null; + int bestDistance = int.MaxValue; + foreach (string candidate in _queried) + { + int distance = Distance(name, candidate); + if (distance < bestDistance) (best, bestDistance) = (candidate, distance); + } + + // A third of the length, so short options do not suggest each other: --threads and + // --report are five edits apart and neither is a plausible typo for the other. + return bestDistance <= Math.Max(1, name.Length / 3) ? best : null; + } + + private static int Distance(string a, string b) + { + var previous = new int[b.Length + 1]; + var current = new int[b.Length + 1]; + for (int j = 0; j <= b.Length; j++) previous[j] = j; + + for (int i = 1; i <= a.Length; i++) + { + current[0] = i; + for (int j = 1; j <= b.Length; j++) + { + int cost = char.ToLowerInvariant(a[i - 1]) == char.ToLowerInvariant(b[j - 1]) ? 0 : 1; + current[j] = Math.Min(Math.Min(current[j - 1] + 1, previous[j] + 1), previous[j - 1] + cost); + } + + (previous, current) = (current, previous); + } + + return previous[b.Length]; + } + + /// + /// Expands file arguments and glob patterns into a sorted, de-duplicated list. Shells + /// that do not expand wildcards (cmd.exe) and shells that do both end up here. + /// + public static List ResolveMzMLFiles(IEnumerable patterns, string? directory) + { + var files = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (string pattern in patterns) + { + if (pattern.Contains('*') || pattern.Contains('?')) + { + string folder = Path.GetDirectoryName(pattern) is { Length: > 0 } d ? d : "."; + string mask = Path.GetFileName(pattern); + if (Directory.Exists(folder)) + { + foreach (string match in Directory.EnumerateFiles(folder, mask)) + { + if (MARS.Pwiz.SpectrumSources.IsReadable(match)) + files.Add(Path.GetFullPath(match)); + } + } + } + else if (File.Exists(pattern)) + { + files.Add(Path.GetFullPath(pattern)); + } + else if (Directory.Exists(pattern) && MARS.Pwiz.SpectrumSources.IsRecognized(pattern)) + { + // Bruker and Agilent runs are directories, not files. A .d named directly is an + // input; a directory that is not a run is handled by --mzml-dir below. + files.Add(Path.GetFullPath(pattern)); + } + else + { + throw new FileNotFoundException($"File not found: {pattern}"); + } + } + + if (!string.IsNullOrEmpty(directory)) + { + if (!Directory.Exists(directory)) + throw new DirectoryNotFoundException($"Directory not found: {directory}"); + // Every format MARS can read, not just mzML - a directory of .raw is as + // legitimate an input as a directory of converted files. + foreach (string match in Directory.EnumerateFiles(directory)) + { + if (MARS.Pwiz.SpectrumSources.IsReadable(match)) + files.Add(Path.GetFullPath(match)); + } + + // Directory-shaped runs inside the directory, e.g. a folder of Bruker .d. + foreach (string match in Directory.EnumerateDirectories(directory)) + { + if (MARS.Pwiz.SpectrumSources.IsReadable(match)) + files.Add(Path.GetFullPath(match)); + } + } + + var sorted = files.ToList(); + sorted.Sort(StringComparer.Ordinal); + return sorted; + } +} diff --git a/dotnet/MARS/CompareCommand.cs b/dotnet/MARS/CompareCommand.cs new file mode 100644 index 0000000..bf7ac16 --- /dev/null +++ b/dotnet/MARS/CompareCommand.cs @@ -0,0 +1,100 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Compares two mzML files on decoded values, for cross-checking the port against the +// Python implementation's output. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using MARS.Core; +using MARS.IO; + +namespace MARS.Cli; + +public static class CompareCommand +{ + public static int Run(CommandLineArgs args) + { + if (args.Flag("help", "h")) + { + Console.Error.WriteLine(""" + Usage: mars compare [options] + + Compares two mzML files on DECODED m/z and intensity values, and reports how + the m/z arrays differ. Byte comparison would be meaningless: two zlib + implementations produce different compressed bytes for identical data. + + Options: + --validate Also check each file's index and checksum + --max-report N Detail lines to print (default 10) + -v, --verbose Verbose output + """); + return Program.ExitSuccess; + } + + Log.Verbose = args.Flag("verbose", "v"); + + if (args.Positional.Count < 2) + { + Log.Error("Two mzML files are required. Usage: mars compare "); + return Program.ExitInputError; + } + + string pathA = args.Positional[0]; + string pathB = args.Positional[1]; + foreach (string path in new[] { pathA, pathB }) + { + if (File.Exists(path)) continue; + Log.Error($"File not found: {path}"); + return Program.ExitInputError; + } + + bool validate = args.Flag("validate"); + int maxReport = args.Int("max-report") ?? 10; + + // Every option this command reads has been read by now, so a typo can be named rather + // than silently ignored. RejectUnknown only knows an option is real because something + // asked for it. + args.RejectUnknown(); + + var stopwatch = Stopwatch.StartNew(); + + if (validate) + { + foreach (string path in new[] { pathA, pathB }) + { + IndexValidationResult validation = MzMLValidator.Validate(path); + Log.Info($"{Path.GetFileName(path)}: index {(validation.BadOffsets.Count == 0 ? "valid" : "INVALID")}" + + $", checksum {(validation.ChecksumPresent ? validation.ChecksumValid ? "valid" : "INVALID" : "absent")}" + + $", {validation.SpectrumOffsets:N0} spectrum offsets"); + } + } + + Log.Info($"Comparing {Path.GetFileName(pathA)} against {Path.GetFileName(pathB)}..."); + MzMLComparison comparison = MzMLComparer.Compare(pathA, pathB, maxReport); + + Console.Out.WriteLine($"spectra compared {comparison.SpectraCompared:N0}"); + Console.Out.WriteLine($"peaks compared {comparison.MzValuesCompared:N0}"); + Console.Out.WriteLine($"m/z values differing {comparison.MzValuesDiffering:N0}"); + Console.Out.WriteLine($"max |delta m/z| {comparison.MaxAbsoluteMzDifference:R} Th"); + Console.Out.WriteLine($"intensity differing {comparison.IntensityValuesDiffering:N0}"); + + if (comparison.SpectraOnlyInA != 0 || comparison.SpectraOnlyInB != 0) + { + Console.Out.WriteLine($"spectra only in A {comparison.SpectraOnlyInA:N0}"); + Console.Out.WriteLine($"spectra only in B {comparison.SpectraOnlyInB:N0}"); + } + + if (comparison.Diverged) + { + Console.Out.WriteLine( + "comparison stopped the files stopped holding the same spectra in the same " + + "order; the counts above cover only what came before that point"); + } + + foreach (string problem in comparison.Problems) Log.Info(" " + problem); + + Log.Info($"Compared in {stopwatch.Elapsed.TotalSeconds:F1} s"); + return Program.ExitSuccess; + } +} diff --git a/dotnet/MARS/CorrectedFileWriter.cs b/dotnet/MARS/CorrectedFileWriter.cs new file mode 100644 index 0000000..fc54ac0 --- /dev/null +++ b/dotnet/MARS/CorrectedFileWriter.cs @@ -0,0 +1,141 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.IO; +using MARS.Core; +using MARS.IO; +using MARS.Pwiz; + +namespace MARS.Cli; + +/// +/// Writes one corrected file, in whichever format was asked for. +/// +/// +/// Two writers sit behind this, and which one runs is decided by the format alone. +/// +/// +/// +/// mzML goes through MARS's own writer, which splices corrected bytes into a copy of +/// the input. Everything MARS did not change is identical to the input by construction rather +/// than by care - see docs/mzml-passthrough.md for why that matters and what broke without +/// it. This is the default and stays the default. +/// +/// +/// +/// +/// Everything else goes through pwiz-sharp, because there is no input of that format +/// to splice into: the file has to be built. That also puts the format code with the people +/// who maintain the format. +/// +/// +/// +/// Both paths run the same over the same values, which is +/// checked rather than assumed - writing a file both ways and diffing with mars compare +/// finds no difference across 82 million peaks. +/// +internal static class CorrectedFileWriter +{ + public static void Write( + MarsOutputFormat format, + ISpectrumSource source, + string outputPath, + MzCalibrator calibrator, + CorrectionOptions correctionOptions, + TemperatureSet? temperatures, + int threads) + { + Log.Info($"Writing: {Path.GetFileName(outputPath)}"); + + // Splice only when there is something to splice into. An mzML input can be copied and + // patched; a vendor file cannot, so its mzML has to be built like any other format. + if (format == MarsOutputFormat.MzML && source is MzMLSpectrumSource mzml) + { + MzMLWriteResult spliced = MzMLWriter.Write( + mzml.Info, + outputPath, + () => new CalibratingTransform(calibrator, correctionOptions, temperatures), + new MzMLWriteOptions { MaxDegreeOfParallelism = threads }, + Log.Warn); + + Report(spliced.SpectraCorrected, spliced.SpectraSeen, spliced.OutputLength, + spliced.MonotonicityFixes, correctionOptions); + return; + } + + PwizWriteResult written = PwizOutput.Write(new PwizWriteRequest + { + InputPath = source.Path, + OutputPath = outputPath, + Format = format, + Calibrator = calibrator, + Options = correctionOptions, + AcquisitionStartTime = source.AcquisitionStartTime, + Temperatures = temperatures, + Threads = threads, + + // Match what the input used, when the input is an mzML that can be read for it. + // pwiz's own default is 64-bit uncompressed, which makes the output substantially + // larger than the file it came from. A vendor file has no encoding to copy, so it + // takes the default, which is what msconvert would have written anyway. + Encoding = SpectrumSources.IsNative(source.Path) + ? MzMLEncoding.Sniff(source.Path) + : SpectrumEncoding.Default, + }); + + Report(written.SpectraCorrected, written.SpectraSeen, written.OutputLength, + written.MonotonicityFixes, correctionOptions); + + // Where the time went, so a slow conversion can be attributed rather than guessed at. + // Everything not in these two is inside pwiz's encoder. + Log.Debug($" reader {written.ReaderTime.TotalSeconds:F1} s, " + + $"model {written.CorrectorTime.TotalSeconds:F1} s"); + + if (written.SpectraReverted > 0) + { + Log.Warn($" {written.SpectraReverted:N0} spectra were left uncorrected because the " + + "correction would have reordered their peaks"); + } + } + + /// The output path for one input, in one format. + public static string OutputPathFor(string inputFile, string outputDirectory, MarsOutputFormat format) => + Path.Combine( + outputDirectory, + Path.GetFileNameWithoutExtension(inputFile) + "-mars" + PwizOutput.Extension(format)); + + /// + /// Resolves --output-format, failing before any work rather than after. + /// + public static MarsOutputFormat ResolveFormat(CommandLineArgs args) + { + string? requested = args.String("output-format"); + if (!PwizOutput.TryParse(requested, out MarsOutputFormat format)) + { + throw new FormatException( + $"--output-format expects mzML, mzXML, mzMLb or mgf, got '{requested}'."); + } + + if (format != MarsOutputFormat.MzML && !PwizOutput.Available) + { + throw new NotSupportedException( + $"This build of MARS cannot write {PwizOutput.Name(format)}: it was built " + + "without a pwiz-sharp checkout. Rebuild with " + + "-p:PwizSharpDir=/pwiz/pwiz-sharp, or write mzML."); + } + + if (PwizOutput.LossWarning(format) is string warning) Log.Warn(warning); + return format; + } + + private static void Report( + long corrected, long seen, long bytes, long monotonicityFixes, CorrectionOptions options) + { + Log.Info($" {corrected:N0} of {seen:N0} spectra corrected, {bytes:N0} bytes"); + if (monotonicityFixes > 0) + { + Log.Warn($" {monotonicityFixes:N0} peaks would have broken ascending m/z order " + + $"and were adjusted ({options.Monotonicity})"); + } + } +} diff --git a/dotnet/MARS/LibrarySource.cs b/dotnet/MARS/LibrarySource.cs new file mode 100644 index 0000000..30eb04e --- /dev/null +++ b/dotnet/MARS/LibrarySource.cs @@ -0,0 +1,91 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using MARS.Core; +using MARS.IO; + +namespace MARS.Cli; + +/// +/// Where the spectral library comes from, read from the command line but not yet opened. +/// +/// +/// Separating "which options were given" from "load it" is what lets both commands read +/// every option they understand before they start work, so +/// can run against a complete picture. It also +/// puts the library-choosing rules in one place: `qc` and `calibrate` have to agree about +/// which file wins and how it is read, or the accuracy `qc` reports is not the accuracy +/// `calibrate` would have found. +/// +public sealed class LibrarySource +{ + private LibrarySource() + { + } + + private string? PrismCsv { get; init; } + + private string? LibraryPath { get; init; } + + private string? DiannReport { get; init; } + + private double RtWindow { get; init; } + + private bool Dedupe { get; init; } + + public static LibrarySource From(CommandLineArgs args) => new() + { + PrismCsv = args.String("prism-csv"), + LibraryPath = args.String("library"), + DiannReport = args.String("diann-report"), + RtWindow = args.Double("rt-window") ?? 0.083, + Dedupe = !args.Flag("no-dedupe-library"), + }; + + /// + /// Sequences are dropped by default because a plate-scale report carries tens of millions + /// of them. A dump is a diagnostic run, so the memory is worth the peptide identity in the + /// output. + /// + public SpectralLibrary Load(List runNames, bool keepSequences, Action log) + { + var options = new PrismLibraryOptions + { + RunNames = runNames, + DedupeFragments = Dedupe, + KeepSequences = keepSequences, + }; + + if (PrismCsv is not null) + { + log($"Loading PRISM library: {PrismCsv}"); + return PrismCsvLibraryReader.Load(PrismCsv, options, log); + } + + if (LibraryPath is null) + throw new FileNotFoundException("A library is required: --prism-csv or --library."); + + if (LibraryPath.EndsWith(".csv", StringComparison.OrdinalIgnoreCase)) + { + log($"Loading PRISM library: {LibraryPath}"); + return PrismCsvLibraryReader.Load(LibraryPath, options, log); + } + + if (LibraryPath.EndsWith(".parquet", StringComparison.OrdinalIgnoreCase)) + { + log($"Loading DIA-NN library: {LibraryPath}"); + return DiannParquetLibraryReader.Load(LibraryPath, DiannReport, runNames, log); + } + + if (LibraryPath.EndsWith(".blib", StringComparison.OrdinalIgnoreCase)) + { + log($"Loading BiblioSpec library: {LibraryPath}"); + return BlibLibraryReader.Load(LibraryPath, RtWindow, log); + } + + throw new InvalidDataException( + $"Unrecognized library type '{Path.GetExtension(LibraryPath)}'. Expected .blib, .parquet or .csv."); + } +} diff --git a/dotnet/MARS/MARS.csproj b/dotnet/MARS/MARS.csproj new file mode 100644 index 0000000..e6fa3d6 --- /dev/null +++ b/dotnet/MARS/MARS.csproj @@ -0,0 +1,64 @@ + + + + + + + Exe + MARS.Cli + mars + Major + MARS (Mass Accuracy Recalibration System) command line tool. + + + + + + + + + + + + + + + + + + <_SciexDeps>$(PwizSharpDir)\vendor-archives\Sciex + <_SciexAssemblies>$(PwizSharpDir)\vendor-assemblies\Sciex + <_Wiff2Dir>$([MSBuild]::EnsureTrailingSlash('$(OutputPath)'))wiff2 + <_Wiff2PublishDir Condition="'$(PublishDir)' != ''">$([MSBuild]::EnsureTrailingSlash('$(PublishDir)'))wiff2 + + + + <_Wiff2Payload Include="$(_SciexDeps)\System.Data.SQLite.dll" /> + <_Wiff2Payload Include="$(_SciexDeps)\Unity.Abstractions.dll" /> + <_Wiff2Payload Include="$(_SciexAssemblies)\SQLite.Interop.dll" /> + + + + + + + + + + diff --git a/dotnet/MARS/Program.cs b/dotnet/MARS/Program.cs new file mode 100644 index 0000000..b582e2d --- /dev/null +++ b/dotnet/MARS/Program.cs @@ -0,0 +1,215 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// MARS command line entry point. + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using MARS.Core; +using MARS.Pwiz; + +namespace MARS.Cli; + +public static class Program +{ + /// Exit codes, as specified for the MARS CLI. + public const int ExitSuccess = 0; + + public const int ExitInputError = 1; + + public const int ExitInsufficientTrainingData = 2; + + public const int ExitOutputValidationFailure = 3; + + public static int Main(string[] args) + { + PinInvariantCulture(); + + // Every diagnostic goes to stderr so stdout stays clean for piping. + return Dispatch(args); + } + + /// + /// Makes every thread format and parse numbers the same way, whatever the machine's + /// locale is. + /// + /// + /// + /// MARS used to get this from InvariantGlobalization, which forces the whole runtime + /// to the invariant culture. That had to be relaxed for builds carrying a vendor reader, + /// because the Thermo SDK constructs CultureInfo("en-US") and throws when cultures + /// are unavailable. Relaxing it hands CurrentCulture back to the operating system, + /// and on a machine set to a locale that writes decimals with a comma, anything formatted + /// or parsed without an explicit culture silently changes meaning. + /// + /// + /// Setting the default culture instead keeps ICU loaded - so the SDK can ask for the + /// culture it wants - while MARS's own numbers stay invariant. That matters for output + /// that is read by other programs rather than by people: SVG coordinates in the QC report, + /// numbers in the model JSON, and the values parsed back out of a BiblioSpec library, all + /// of which would be corrupted rather than merely ugly. + /// + /// + private static void PinInvariantCulture() + { + CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture; + CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture; + CultureInfo.CurrentCulture = CultureInfo.InvariantCulture; + CultureInfo.CurrentUICulture = CultureInfo.InvariantCulture; + } + + private static int Dispatch(string[] args) + { + if (args.Length == 0 || args[0] is "-h" or "--help" or "help") + { + PrintUsage(); + return args.Length == 0 ? ExitInputError : ExitSuccess; + } + + if (args[0] is "--version" or "-V") + { + Console.Out.WriteLine(MarsInfo.Version); + + // What this particular binary can do, not what MARS can do in principle. Vendor + // reading and the non-mzML outputs depend on whether the build had a pwiz-sharp + // checkout, and there is otherwise no way to tell two identically named binaries + // apart until one of them refuses a file. + Console.Out.WriteLine( + "reads: " + string.Join(", ", SpectrumSources.ReadableExtensions())); + Console.Out.WriteLine( + "writes: " + string.Join(", ", PwizOutput.Supported.Select(PwizOutput.Name))); + return ExitSuccess; + } + + CommandLineArgs parsed; + try + { + parsed = CommandLineArgs.Parse(args); + } + catch (Exception ex) + { + Log.Error(ex.Message); + return ExitInputError; + } + + try + { + int exit = parsed.Command.ToLowerInvariant() switch + { + "verify" => VerifyCommand.Run(parsed), + "calibrate" => CalibrateCommand.Run(parsed), + "apply" => ApplyCommand.Run(parsed), + "qc" => QcCommand.Run(parsed), + "compare" => CompareCommand.Run(parsed), + _ => UnknownCommand(parsed.Command), + }; + + // Commands that finish their option reading call RejectUnknown() themselves, + // before doing any work. This is the backstop for the ones that return early - + // an option read after an early return was never queried, so it cannot be + // distinguished from a typo here and stays a warning rather than an error. + foreach (string unknown in parsed.UnknownOptions()) + Log.Warn($"Unrecognized option --{unknown} was ignored."); + + return exit; + } + catch (UnknownOptionException ex) + { + Log.Error(ex.Message); + return ExitInputError; + } + catch (FileNotFoundException ex) + { + Log.Error(ex.Message); + return ExitInputError; + } + catch (DirectoryNotFoundException ex) + { + Log.Error(ex.Message); + return ExitInputError; + } + catch (InsufficientTrainingDataException ex) + { + Log.Error(ex.Message); + return ExitInsufficientTrainingData; + } + catch (OutputValidationException ex) + { + Log.Error(ex.Message); + return ExitOutputValidationFailure; + } + catch (Exception ex) + { + Log.Error(ex.Message); + if (Log.Verbose) Log.Error(ex.ToString()); + return ExitInputError; + } + } + + private static int UnknownCommand(string command) + { + Log.Error($"Unknown command '{command}'."); + PrintUsage(); + return ExitInputError; + } + + private static void PrintUsage() + { + Console.Error.WriteLine($""" + MARS {MarsInfo.Version} - Mass Accuracy Recalibration System + + Usage: mars [options] + + Commands: + calibrate Learn an m/z calibration from spectral library matches and write + recalibrated mzML files. + apply Apply a previously trained model to more files. + qc Report current mass accuracy without training or writing. + verify Round-trip a file through the passthrough writer with a null + correction, then check the index, checksum and decoded arrays. + compare Compare two mzML files on decoded m/z and intensity values. + + Run 'mars --help' for the options of a command. + """); + } +} + +/// Fewer usable training rows than the run needs to fit anything meaningful. +public sealed class InsufficientTrainingDataException : Exception +{ + public InsufficientTrainingDataException(string message) + : base(message) + { + } +} + +/// A written file failed its structural checks. +public sealed class OutputValidationException : Exception +{ + public OutputValidationException(string message) + : base(message) + { + } +} + +internal static class Log +{ + public static bool Verbose { get; set; } + + public static void Info(string message) => + Console.Error.WriteLine($"{Timestamp()} INFO {message}"); + + public static void Warn(string message) => + Console.Error.WriteLine($"{Timestamp()} WARN {message}"); + + public static void Error(string message) => + Console.Error.WriteLine($"{Timestamp()} ERROR {message}"); + + public static void Debug(string message) + { + if (Verbose) Console.Error.WriteLine($"{Timestamp()} DEBUG {message}"); + } + + private static string Timestamp() => + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); +} diff --git a/dotnet/MARS/QcCommand.cs b/dotnet/MARS/QcCommand.cs new file mode 100644 index 0000000..e50e62a --- /dev/null +++ b/dotnet/MARS/QcCommand.cs @@ -0,0 +1,299 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Ported from the qc command in mars/cli.py: report current mass accuracy without +// training a model or writing any file. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using MARS.Core; +using MARS.IO; +using MARS.Pwiz; +using MARS.Report; + +namespace MARS.Cli; + +public static class QcCommand +{ + public static int Run(CommandLineArgs args) + { + if (args.Flag("help", "h")) + { + Console.Error.WriteLine(""" + Usage: mars qc [options] [ ...] + + Matches library fragments and reports the mass accuracy already present in + the files. Trains nothing and writes no mzML. + + Options: + --mzml mzML file or glob (repeatable) + --mzml-dir Directory of mzML files + --prism-csv Skyline PRISM report CSV + --library .blib, report-lib.parquet, or PRISM .csv + --diann-report DIA-NN report.parquet + --resolution unit, hram or auto (default auto: read the + mass analyzer from the mzML and pick) + --tolerance Fragment tolerance in Th (default 0.3) + --tolerance-ppm Fragment tolerance in ppm (default 10 on + high-resolution data) + --min-intensity Minimum peak intensity (default 500) + --max-isolation-window + Skip wider isolation windows + --temperature-dir Directory of RFA2-/RFC2- temperature CSVs + --output Report path (default mars_qc_summary.txt) + --html-report Where to write the figures (default + mars_qc_report.html beside the summary). + One self-contained file, safe to email + --no-html-report Skip the figures and write only the summary + --by-file Report each input file separately + -v, --verbose Verbose output + """); + return Program.ExitSuccess; + } + + Log.Verbose = args.Flag("verbose", "v"); + + var patterns = new List(args.Strings("mzml", "mzML")); + patterns.AddRange(args.Positional); + List mzmlFiles = CommandLineArgs.ResolveMzMLFiles(patterns, args.String("mzml-dir")); + if (mzmlFiles.Count == 0) + { + Log.Error("No mzML files found. Use --mzml, --mzml-dir, or pass files as arguments."); + return Program.ExitInputError; + } + + var matchOptions = new MatchOptions + { + MzToleranceTh = args.Double("tolerance") ?? ResolutionMode.DefaultToleranceTh, + TolerancePpm = args.Double("tolerance-ppm") ?? 0, + MinIntensity = args.Double("min-intensity") ?? 500.0, + MaxIsolationWindowWidth = args.Double("max-isolation-window"), + }; + + + + string reportPath = args.String("output") ?? "mars_qc_summary.txt"; + bool byFile = args.Flag("by-file"); + bool noHtmlReport = args.Flag("no-html-report"); + string htmlReportPath = args.String("html-report") ?? DefaultHtmlPath(reportPath); + string? temperatureDirectory = args.String("temperature-dir"); + + // Read before the check below rather than inside LoadLibrary, so that every option + // this command understands has been seen by the time the check runs. + var librarySource = LibrarySource.From(args); + + // Everything is read; refuse a typo now rather than after minutes of work. + // Read here, used further down once the readers are open and can say what analyzer + // they saw. RejectUnknown only knows an option is real because something asked for it, + // so an option resolved later has to be touched before the check or it is reported as + // a typo. + ResolutionMode.Touch(args); + + args.RejectUnknown(); + + var runNames = new List(); + foreach (string file in mzmlFiles) runNames.Add(Path.GetFileName(file)); + + SpectralLibrary library = librarySource.Load(runNames, keepSequences: false, Log.Info); + var stopwatch = Stopwatch.StartNew(); + + var text = new StringBuilder(); + text.AppendLine("Mars QC Report (pre-calibration)"); + text.AppendLine(new string('=', 40)); + text.AppendLine(); + text.AppendLine($"Files: {mzmlFiles.Count}"); + text.AppendLine($"Tolerance: {(matchOptions.TolerancePpm > 0 ? $"+/-{matchOptions.TolerancePpm:F1} ppm" : $"+/-{matchOptions.MzToleranceTh:F3} Th")}"); + text.AppendLine($"Minimum intensity: {matchOptions.MinIntensity:N0}"); + text.AppendLine(); + + // Reporting accuracy needs only two features. The figures want every feature there + // is, since a panel per feature is most of their value, and computing them costs one + // pass over peaks MARS has already decoded. Collect the wider set only when the + // figures are actually going to be drawn. + var sourceByFile = new Dictionary(StringComparer.OrdinalIgnoreCase); + var temperatureByFile = new Dictionary(StringComparer.OrdinalIgnoreCase); + bool anyRfa2 = false, anyRfc2 = false; + + foreach (string file in mzmlFiles) + { + sourceByFile[file] = SpectrumSources.Open(file); + if (temperatureDirectory is null) continue; + + TemperatureSet temperatures = TemperatureCsvReader.Find(file, temperatureDirectory, Log.Info); + temperatureByFile[file] = temperatures; + anyRfa2 |= temperatures.Rfa2 is not null; + anyRfc2 |= temperatures.Rfc2 is not null; + } + + // Decided once the readers are open, from what the first of them says its MS2 + // analyzer is. The readers know their own formats; asking the file again from here + // would mean parsing a .raw as if it were mzML, which is how this used to fall back + // to a trap tolerance on Astral data without anyone noticing. + ResolutionMode resolution = ResolutionMode.Resolve( + args, sourceByFile[mzmlFiles[0]].Analyzer, matchOptions, Log.Info); + + MarsFeature[] collect; + if (noHtmlReport) + { + // Only fragment m/z. The summary needs it to convert each row to ppm, and with + // no figures to draw nothing reads any other feature column. + collect = new[] { MarsFeature.FragmentMz }; + } + else + { + InjectionTimeUse injectionTime = CalibrateCommand.ReportInjectionTime( + CalibrateCommand.ProbeInjectionTime(sourceByFile[mzmlFiles[0]])); + collect = FragmentMatcher.CollectedFeatures(injectionTime, anyRfa2, anyRfc2); + } + + var combined = new MatchTable(collect, keepDetail: !noHtmlReport); + var matcher = new FragmentMatcher(library, matchOptions); + + foreach (string file in mzmlFiles) + { + ISpectrumSource source = sourceByFile[file]; + int rowsBefore = combined.Count; + temperatureByFile.TryGetValue(file, out TemperatureSet? temperatures); + + Log.Info($"Matching: {Path.GetFileName(file)}"); + foreach (SpectrumRecord spectrum in source.ReadSpectra(msLevel: 2)) + matcher.MatchSpectrum(spectrum, temperatures, combined); + + Log.Info($" {combined.Count - rowsBefore:N0} fragment matches"); + + // Per-file numbers come from this file's slice of the shared table, so each + // spectrum is still matched exactly once. + if (byFile && combined.Count > rowsBefore) + { + text.AppendLine(Path.GetFileName(file)); + AppendSummary(text, combined, rowsBefore, combined.Count - rowsBefore); + text.AppendLine(); + } + } + + CalibrateCommand.CheckTolerance(combined, matchOptions); + + if (combined.Count == 0) + { + Log.Error("No fragment matches found. Check that the library describes these runs."); + return Program.ExitInsufficientTrainingData; + } + + text.AppendLine("All files"); + AppendSummary(text, combined, 0, combined.Count); + + string? directory = Path.GetDirectoryName(Path.GetFullPath(reportPath)); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + File.WriteAllText(reportPath, text.ToString()); + + Console.Out.Write(text.ToString()); + + if (!noHtmlReport) + { + QcHtmlReport.Write( + htmlReportPath, + BuildReportData(combined, collect), + statistics: null, + matcher.Statistics, + mzmlFiles, + matchOptions.TolerancePpm > 0 + ? $"{matchOptions.TolerancePpm:0.##} ppm" + : $"{matchOptions.MzToleranceTh:0.###} Th", + MarsInfo.Version, + Uncorrected(combined, resolution), + resolution.ReportInPpm ? ErrorScale.Ppm : ErrorScale.Th); + Log.Info($"Wrote QC figures to {htmlReportPath}"); + } + + foreach (ISpectrumSource source in sourceByFile.Values) source.Dispose(); + + Log.Info($"Wrote {reportPath} in {stopwatch.Elapsed.TotalSeconds:F1} s"); + return Program.ExitSuccess; + } + + /// Puts the figures next to the text report rather than the working directory. + private static string DefaultHtmlPath(string reportPath) + { + string? directory = Path.GetDirectoryName(Path.GetFullPath(reportPath)); + return string.IsNullOrEmpty(directory) + ? "mars_qc_report.html" + : Path.Combine(directory, "mars_qc_report.html"); + } + + /// + /// The measured error, summarized on the scale the report will be drawn in. + /// + private static ErrorSummary Uncorrected(MatchTable table, ResolutionMode resolution) + { + ReadOnlySpan delta = table.DeltaMz.Items.AsSpan(0, table.Count); + if (!resolution.ReportInPpm) return MarsStatistics.Summarize(delta); + + double[] fragmentMz = table.Column(MarsFeature.FragmentMz).Items; + var ppm = new double[table.Count]; + for (int i = 0; i < ppm.Length; i++) + { + double mz = fragmentMz[i]; + ppm[i] = mz > 0 ? delta[i] / mz * 1e6 : 0.0; + } + + return MarsStatistics.Summarize(ppm); + } + + /// + /// Collects the per-row values the figures are drawn from. There is no model here, so + /// there is no corrected error and no importance; the report renders the measured error + /// alone rather than pretending to a before-and-after. + /// + private static QcHtmlReport.Data BuildReportData(MatchTable table, MarsFeature[] collected) + { + int rows = table.Count; + var features = new List<(string Name, double[] Values)>(collected.Length); + foreach (MarsFeature feature in collected) + features.Add((MarsFeatures.NameOf(feature), table.Column(feature).Items[..rows])); + + return new QcHtmlReport.Data + { + ErrorBefore = table.DeltaMz.Items[..rows], + ErrorAfter = Array.Empty(), + RetentionTime = table.RetentionTime is null ? Array.Empty() : table.RetentionTime.Items[..rows], + FragmentMz = table.Has(MarsFeature.FragmentMz) + ? table.Column(MarsFeature.FragmentMz).Items[..rows] + : Array.Empty(), + Features = features, + ImportanceNames = Array.Empty(), + Importance = Array.Empty(), + }; + } + + private static void AppendSummary(StringBuilder text, MatchTable table, int start, int count) + { + ReadOnlySpan delta = table.DeltaMz.Items.AsSpan(start, count); + ErrorSummary summary = MarsStatistics.Summarize(delta); + + // Converted per row from that fragment's own m/z, not by dividing the aggregate by a + // nominal mass - the fragments here span most of a factor of four in m/z, so the + // shortcut would be wrong by about that much. + double[] fragmentMz = table.Column(MarsFeature.FragmentMz).Items; + var ppm = new double[count]; + for (var i = 0; i < count; i++) + { + double mz = fragmentMz[start + i]; + ppm[i] = mz > 0 ? delta[i] / mz * 1e6 : 0.0; + } + + ErrorSummary p = MarsStatistics.Summarize(ppm); + + // Both scales, in the same layout calibrate uses. Th is what an ion trap is specified + // in; ppm is the scale a high-resolution instrument is specified in and the only one + // that compares across instruments. + text.AppendLine($" Matches: {summary.Count:N0}"); + text.AppendLine($" Mean delta: {summary.Mean,9:F4} Th {p.Mean,8:F2} ppm"); + text.AppendLine($" Median delta: {summary.Median,9:F4} Th {p.Median,8:F2} ppm"); + text.AppendLine($" Std delta: {summary.StdDev,9:F4} Th {p.StdDev,8:F2} ppm"); + text.AppendLine($" MAD delta: {summary.Mad,9:F4} Th {p.Mad,8:F2} ppm"); + text.AppendLine($" RMS delta: {summary.Rms,9:F4} Th {p.Rms,8:F2} ppm"); + text.AppendLine($" MAE delta: {summary.Mae,9:F4} Th {p.Mae,8:F2} ppm"); + } + +} diff --git a/dotnet/MARS/QcReport.cs b/dotnet/MARS/QcReport.cs new file mode 100644 index 0000000..9a5ea62 --- /dev/null +++ b/dotnet/MARS/QcReport.cs @@ -0,0 +1,241 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// The QC path's text report. Layout follows the Python mars_qc_summary.txt so the two can +// be diffed directly during the port. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using MARS.Core; + +namespace MARS.Cli; + +public static class QcReport +{ + public static void Write( + string path, + MzCalibrator calibrator, + MatchStatistics matchStatistics, + IReadOnlyList inputFiles, + MatchOptions matchOptions) + { + TrainingStatistics stats = calibrator.Statistics + ?? throw new InvalidOperationException("Model carries no training statistics."); + + var text = new StringBuilder(); + text.AppendLine("Mars Calibration QC Summary"); + text.AppendLine(new string('=', 50)); + text.AppendLine(); + + text.AppendLine("Input:"); + foreach (string file in inputFiles) text.AppendLine(" " + Path.GetFileName(file)); + text.AppendLine($" Tolerance: {DescribeTolerance(matchOptions)}"); + text.AppendLine($" Minimum intensity: {matchOptions.MinIntensity:N0}"); + if (matchOptions.MaxIsolationWindowWidth is double width) + text.AppendLine($" Maximum isolation window: {width:F2} Th"); + text.AppendLine(); + + text.AppendLine("Before Calibration:"); + AppendSummary(text, stats.Before, stats.BeforePpm); + text.AppendLine(); + + text.AppendLine(calibrator.CrossValidation is null + ? "After Calibration:" + : "After Calibration (these files, corrected):"); + AppendSummary(text, stats.After, stats.AfterPpm); + text.AppendLine(); + + text.AppendLine($"Improvement: {Reduction(stats.Before.StdDev, stats.After.StdDev):F1}% reduction in std dev"); + text.AppendLine($" {Reduction(stats.Before.Mad, stats.After.Mad):F1}% reduction in MAD"); + text.AppendLine($" {Reduction(stats.Before.Rms, stats.After.Rms):F1}% reduction in RMS"); + if (stats.BeforePpm is ErrorSummary bp && stats.AfterPpm is ErrorSummary ap) + { + text.AppendLine( + $" MAD {bp.Mad:F2} -> {ap.Mad:F2} ppm, " + + $"median {bp.Median:+0.00;-0.00} -> {ap.Median:+0.00;-0.00} ppm"); + } + text.AppendLine(); + + if (calibrator.CrossValidation is CrossValidationReport estimate) + { + // The two numbers answer different questions and a reader deserves both. The + // figures above describe these files. This one describes what the same procedure + // would achieve on a run it was not fitted to, which is what `mars apply` does. + text.AppendLine( + $"Expected on data not used to fit: MAD {estimate.OutOfFold.Mad:F4} Th" + + (estimate.OutOfFoldPpm is FoldMetrics oofPpm ? $" ({oofPpm.Mad:F2} ppm)" : string.Empty) + + $", {estimate.OutOfFold.MadReduction:F1}% reduction, from cross-validation below."); + } + + text.AppendLine(); + + if (calibrator.CrossValidation is CrossValidationReport cv) + { + text.AppendLine("Cross-Validation (folds split by peptide)"); + text.AppendLine(new string('=', 40)); + text.AppendLine($"Folds: {cv.Folds} over {cv.Groups:N0} peptides"); + text.AppendLine(); + bool ppm = cv.PerFoldPpm is not null; + text.AppendLine(ppm + ? " fold rows MAD Th MAD ppm RMS ppm reduction Pearson r" + : " fold rows MAD Th RMS Th reduction Pearson r"); + for (int i = 0; i < cv.PerFold.Length; i++) + { + FoldMetrics fold = cv.PerFold[i]; + text.AppendLine(ppm + ? $" {i + 1,4} {fold.Rows,10:N0} {fold.Mad,10:F4} {cv.PerFoldPpm![i].Mad,10:F2}" + + $" {cv.PerFoldPpm[i].Rms,10:F2} {fold.MadReduction,9:F1}% {fold.PearsonR,10:F4}" + : $" {i + 1,4} {fold.Rows,10:N0} {fold.Mad,10:F4} {fold.Rms,10:F4}" + + $" {fold.MadReduction,9:F1}% {fold.PearsonR,10:F4}"); + } + + text.AppendLine(); + text.AppendLine( + $" pooled out-of-fold: MAD {cv.OutOfFold.Mad:F4} Th" + + (cv.OutOfFoldPpm is FoldMetrics op ? $" ({op.Mad:F2} ppm)" : string.Empty) + + $", RMS {cv.OutOfFold.Rms:F4} Th, r {cv.OutOfFold.PearsonR:F4}"); + text.AppendLine( + $" spread across folds: MAD {cv.MadSpread:F4} Th, RMS {cv.RmsSpread:F4} Th, " + + $"r {cv.PearsonRSpread:F4}"); + text.AppendLine( + $" on this data: MAD {cv.InSample.Mad:F4} Th" + + (cv.InSamplePpm is FoldMetrics ip ? $" ({ip.Mad:F2} ppm)" : string.Empty) + + $"; gap to the estimate above {cv.OptimismMad:F4} Th ({OptimismVerdict(cv)})"); + text.AppendLine(); + text.AppendLine(" Every row above was scored by a model that never saw its peptide,"); + text.AppendLine(" so this is the estimate for a run the model was not fitted to."); + text.AppendLine(" The figures at the top of this report describe THESE files, which the"); + text.AppendLine(" applied model was fitted to - as mass calibration normally is."); + text.AppendLine($" {DescribeSpread(cv)}"); + text.AppendLine(); + text.AppendLine(); + } + + text.AppendLine("Calibration Model Summary"); + text.AppendLine(new string('=', 40)); + text.AppendLine($"Matched fragments: {stats.RowsMatched:N0}"); + text.AppendLine($"Training samples: {stats.RowsUsed:N0}"); + text.AppendLine(calibrator.CrossValidation is null + ? $"Train/Val split: {stats.RowsTrain:N0} / {stats.RowsValidation:N0}" + : $"Model: fitted on all {stats.RowsUsed:N0} rows; " + + $"{calibrator.CrossValidation.Folds}-fold cross-validation run alongside"); + text.AppendLine($"Spectra examined: {matchStatistics.SpectraSeen:N0}"); + text.AppendLine($"Library precursors matched: {matchStatistics.UniqueEntriesMatched:N0}"); + text.AppendLine(); + + text.AppendLine("Model performance:"); + bool crossValidated = calibrator.CrossValidation is not null; + text.AppendLine($" {(crossValidated ? "On this data MAE: " : "Train MAE: ")}{stats.TrainMae:F4} Th"); + text.AppendLine($" {(crossValidated ? "On this data RMSE:" : "Train RMSE: ")}{stats.TrainRmse:F4} Th"); + if (stats.RowsValidation > 0) + { + text.AppendLine($" {(crossValidated ? "Out-of-fold MAD: " : "Val MAE: ")}{stats.ValidationMae:F4} Th"); + text.AppendLine($" {(crossValidated ? "Out-of-fold RMSE: " : "Val RMSE: ")}{stats.ValidationRmse:F4} Th"); + } + + text.AppendLine(); + text.AppendLine("Hyperparameters:"); + text.AppendLine($" n_estimators: {calibrator.Options.NEstimators}"); + text.AppendLine($" max_depth: {calibrator.Options.MaxDepth}"); + text.AppendLine($" learning_rate: {calibrator.Options.LearningRate.ToString("R", CultureInfo.InvariantCulture)}"); + text.AppendLine($" min_child_weight: {calibrator.Options.MinChildWeight.ToString("R", CultureInfo.InvariantCulture)}"); + text.AppendLine($" subsample: {calibrator.Options.Subsample.ToString("R", CultureInfo.InvariantCulture)}"); + text.AppendLine($" colsample: {calibrator.Options.ColSampleByTree.ToString("R", CultureInfo.InvariantCulture)}"); + text.AppendLine($" reg_lambda: {calibrator.Options.RegLambda.ToString("R", CultureInfo.InvariantCulture)}"); + text.AppendLine($" max_bin: {calibrator.Options.MaxBins}"); + text.AppendLine($" seed: {calibrator.Options.Seed}"); + text.AppendLine(); + + text.AppendLine("Feature importance (permutation, normalized):"); + string[] names = calibrator.Features.Names(); + for (int i = 0; i < names.Length; i++) + { + double importance = i < stats.PermutationImportance.Length ? stats.PermutationImportance[i] : 0.0; + int splits = i < stats.SplitCount.Length ? stats.SplitCount[i] : 0; + text.AppendLine($" {names[i]}: {importance:F3} ({splits:N0} splits)"); + } + + text.AppendLine(); + text.AppendLine("Note: importance is permutation-based (the rise in RMSE when a feature is"); + text.AppendLine("shuffled), not XGBoost's gain. Values are not comparable term by term with"); + text.AppendLine("the Python report; the ranking is."); + + string? directory = Path.GetDirectoryName(Path.GetFullPath(path)); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + File.WriteAllText(path, text.ToString()); + } + + /// + /// Reads the fold-to-fold spread relative to the accuracy being reported. A spread is + /// only meaningful against the size of the thing it varies around. + /// + private static string DescribeSpread(CrossValidationReport cv) + { + if (cv.PerFold.Length < 2 || !(cv.OutOfFold.Mad > 0) || double.IsNaN(cv.MadSpread)) + return "Too few folds to judge how much the estimate varies."; + + double relative = cv.MadSpread / cv.OutOfFold.Mad; + return relative switch + { + < 0.05 => "The folds agree closely, so the pooled figure is a stable estimate.", + < 0.15 => "The folds vary a little; the pooled figure is a reasonable estimate.", + _ => "The folds disagree substantially, so the pooled figure is an average over " + + "populations the model handles differently rather than a description of any " + + "one of them.", + }; + } + + /// + /// Plain-language reading of the gap between in-sample and out-of-fold accuracy. The + /// number alone does not say whether it is large, and "large" here is relative to the + /// error being corrected rather than absolute. + /// + private static string OptimismVerdict(CrossValidationReport cv) + { + if (!(cv.OutOfFold.Mad > 0)) return "not assessable"; + + double relative = cv.OptimismMad / cv.OutOfFold.Mad; + return relative switch + { + < 0.05 => "negligible; the fit is driven by the instrument, not these peptides", + < 0.15 => "modest", + < 0.30 => "substantial: the fit leans on the particular peptides in this run", + _ => "large: the fit is thin, and reusing this model elsewhere would disappoint", + }; + } + + private static void AppendSummary(StringBuilder text, ErrorSummary summary, ErrorSummary? ppm) + { + text.AppendLine($" Matches: {summary.Count:N0}"); + + // Both scales, always. Th is what an ion trap is specified in and ppm is what a + // high-resolution instrument is specified in, and the same file can be read by + // people who think in either. + if (ppm is ErrorSummary p) + { + text.AppendLine($" Mean delta: {summary.Mean,9:F4} Th {p.Mean,8:F2} ppm"); + text.AppendLine($" Median delta: {summary.Median,9:F4} Th {p.Median,8:F2} ppm"); + text.AppendLine($" Std delta: {summary.StdDev,9:F4} Th {p.StdDev,8:F2} ppm"); + text.AppendLine($" MAD delta: {summary.Mad,9:F4} Th {p.Mad,8:F2} ppm"); + text.AppendLine($" RMS delta: {summary.Rms,9:F4} Th {p.Rms,8:F2} ppm"); + text.AppendLine($" MAE delta: {summary.Mae,9:F4} Th {p.Mae,8:F2} ppm"); + return; + } + + text.AppendLine($" Mean delta m/z: {summary.Mean:F4} Th"); + text.AppendLine($" Median delta m/z: {summary.Median:F4} Th"); + text.AppendLine($" Std delta m/z: {summary.StdDev:F4} Th"); + text.AppendLine($" MAD delta m/z: {summary.Mad:F4} Th"); + text.AppendLine($" RMS delta m/z: {summary.Rms:F4} Th"); + text.AppendLine($" MAE delta m/z: {summary.Mae:F4} Th"); + } + + private static double Reduction(double before, double after) => + before > 0 ? (before - after) / before * 100.0 : 0.0; + + private static string DescribeTolerance(MatchOptions options) => + options.TolerancePpm > 0 + ? $"+/-{options.TolerancePpm:F1} ppm" + : $"+/-{options.MzToleranceTh:F3} Th"; +} diff --git a/dotnet/MARS/Report/Axis.cs b/dotnet/MARS/Report/Axis.cs new file mode 100644 index 0000000..71988e7 --- /dev/null +++ b/dotnet/MARS/Report/Axis.cs @@ -0,0 +1,99 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Linear axis scaling and tick selection for the QC charts. + +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace MARS.Report; + +/// +/// Maps a data range onto a pixel range and chooses readable tick positions. +/// +public readonly struct Axis +{ + public Axis(double min, double max, double pixelLow, double pixelHigh, bool invert = false) + { + // A degenerate range would divide by zero and put every point in one place. Widen it + // around the value so a constant column still renders something sensible. + if (!(max > min)) + { + double pad = Math.Abs(min) > 0 ? Math.Abs(min) * 0.05 : 0.5; + min -= pad; + max += pad; + } + + Min = min; + Max = max; + PixelLow = pixelLow; + PixelHigh = pixelHigh; + Invert = invert; + } + + public double Min { get; } + + public double Max { get; } + + public double PixelLow { get; } + + public double PixelHigh { get; } + + /// True for the y axis, where larger values are drawn higher up the page. + public bool Invert { get; } + + public double Map(double value) + { + double fraction = (value - Min) / (Max - Min); + return Invert + ? PixelHigh - (fraction * (PixelHigh - PixelLow)) + : PixelLow + (fraction * (PixelHigh - PixelLow)); + } + + /// + /// Tick positions at 1, 2 or 5 times a power of ten, which is what makes an axis + /// readable at a glance rather than a row of arbitrary decimals. + /// + public IReadOnlyList Ticks(int target = 5) + { + var ticks = new List(); + double span = Max - Min; + if (span <= 0 || double.IsNaN(span) || double.IsInfinity(span)) return ticks; + + double rough = span / Math.Max(1, target); + double magnitude = Math.Pow(10, Math.Floor(Math.Log10(rough))); + double normalized = rough / magnitude; + double step = normalized switch + { + <= 1 => 1, + <= 2 => 2, + <= 5 => 5, + _ => 10, + } * magnitude; + + double first = Math.Ceiling(Min / step) * step; + for (double t = first; t <= Max + (step * 1e-9); t += step) + { + // Snap values that are a hair off a round number by accumulated error. + double snapped = Math.Abs(t) < step * 1e-9 ? 0 : t; + ticks.Add(snapped); + if (ticks.Count > 40) break; + } + + return ticks; + } + + /// Formats a tick so the label carries the precision the step needs, and no more. + public string Format(double value) + { + double span = Max - Min; + if (span == 0) return value.ToString("0.###", CultureInfo.InvariantCulture); + + double magnitude = Math.Max(Math.Abs(Min), Math.Abs(Max)); + if (magnitude >= 100000 || (magnitude > 0 && magnitude < 0.001)) + return value.ToString("0.##e+0", CultureInfo.InvariantCulture); + + int decimals = Math.Max(0, (int)Math.Ceiling(-Math.Log10(span / 5)) + 1); + decimals = Math.Min(decimals, 6); + return value.ToString("F" + decimals.ToString(CultureInfo.InvariantCulture), CultureInfo.InvariantCulture); + } +} diff --git a/dotnet/MARS/Report/Charts.cs b/dotnet/MARS/Report/Charts.cs new file mode 100644 index 0000000..358e467 --- /dev/null +++ b/dotnet/MARS/Report/Charts.cs @@ -0,0 +1,800 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// The four chart types the QC report uses. + +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace MARS.Report; + +/// Chart rendering for the QC report. Every method returns a standalone SVG. +public static class Charts +{ + private const int Width = 760; + private const int Height = 340; + private const int Left = 66; + private const int Right = 18; + private const int Top = 34; + private const int Bottom = 46; + + private const string Grid = "var(--grid)"; + private const string Axis0 = "var(--axis)"; + private const string Muted = "var(--muted)"; + private const string Before = "#d2695a"; + private const string After = "#3f7fbf"; + + /// + /// Histogram of the mass error before and after correction, overlaid. + /// + /// This is the headline figure: if the after distribution is not visibly narrower than + /// the before one, nothing else in the report matters. + /// + public static string ErrorHistogram( + ReadOnlySpan before, ReadOnlySpan after, string unit, int bins = 160) + { + var svg = new Svg(Width, Height); + if (before.Length == 0) return Empty(svg, "No matched fragments."); + + // A few extreme rows should not squash the informative part of the axis into a + // sliver, so bound the range at a high percentile of the uncorrected error. + double limit = SymmetricLimit(before, 0.995); + var x = new Axis(-limit, limit, Left, Width - Right); + + int[] beforeCounts = Bin(before, -limit, limit, bins); + int[] afterCounts = after.Length > 0 ? Bin(after, -limit, limit, bins) : Array.Empty(); + + int peak = 0; + foreach (int c in beforeCounts) peak = Math.Max(peak, c); + foreach (int c in afterCounts) peak = Math.Max(peak, c); + var y = new Axis(0, peak <= 0 ? 1 : peak, Top, Height - Bottom, invert: true); + + Frame(svg, x, y, $"mass error ({unit})", "fragments"); + + DrawBars(svg, beforeCounts, x, y, -limit, limit, Before, 0.55); + if (afterCounts.Length > 0) DrawBars(svg, afterCounts, x, y, -limit, limit, After, 0.55); + + // Zero is the whole point of the figure; make it unmissable. + double zero = x.Map(0); + svg.Line(zero, Top, zero, Height - Bottom, Axis0, 1, "3 3"); + + Legend(svg, after.Length > 0); + return svg.ToString(); + } + + /// + /// Median mass error over retention time and fragment m/z, before and after correction, + /// side by side on a shared color scale. + /// + /// + /// + /// Side by side sharing one scale is the point: the question is whether the structure in + /// the left panel is gone from the right, and two panels scaled independently cannot + /// answer it - a faint residue would be stretched to look like the original. + /// + /// + /// The scale comes from the cell medians actually drawn, not from the raw per-row error. + /// A cell median over hundreds of rows is far tighter than the rows it averages, so + /// scaling it against the raw spread washes the field out to near-white, which is + /// exactly what an earlier version of this chart did. + /// + /// + public static string ErrorHeatmapPair( + ReadOnlySpan retentionTime, ReadOnlySpan fragmentMz, + ReadOnlySpan before, ReadOnlySpan after, string unit, + int xBins = 44, int yBins = 36, int minimumPerCell = 8) + { + const int height = 400; + var svg = new Svg(Width, height); + if (before.Length == 0) return Empty(svg, "No matched fragments."); + + bool paired = after.Length == before.Length; + (double rtMin, double rtMax) = Range(retentionTime); + (double mzMin, double mzMax) = Range(fragmentMz); + + double?[] beforeCells = CellMedians( + retentionTime, fragmentMz, before, rtMin, rtMax, mzMin, mzMax, xBins, yBins, minimumPerCell); + double?[]? afterCells = paired + ? CellMedians(retentionTime, fragmentMz, after, rtMin, rtMax, mzMin, mzMax, xBins, yBins, minimumPerCell) + : null; + + // One scale for both panels, taken from the uncorrected field so the corrected one is + // measured against it rather than against itself. + double scale = CellScale(beforeCells); + + const int top = 46; + const int bottom = 64; + const int gap = 24; + const int barWidth = 52; + int panelWidth = paired + ? (Width - Left - Right - gap - barWidth) / 2 + : Width - Left - Right - barWidth; + int panelHeight = height - top - bottom; + + DrawPanel(svg, beforeCells, xBins, yBins, scale, Left, top, panelWidth, panelHeight, + rtMin, rtMax, mzMin, mzMax, paired ? "Before correction" : "As measured", showY: true); + + if (paired) + { + DrawPanel(svg, afterCells!, xBins, yBins, scale, Left + panelWidth + gap, top, + panelWidth, panelHeight, rtMin, rtMax, mzMin, mzMax, "After correction", showY: false); + } + + VerticalColorBar(svg, Width - Right - barWidth + 10, top, panelHeight, scale, unit); + svg.Text(13, top + (panelHeight / 2.0), "fragment m/z", anchor: "middle", size: 15, rotate: -90); + return svg.ToString(); + } + + /// Median of each cell, or null where too few rows landed in it. + /// + /// A cell holding one or two rows has a median that is just those rows, so coloring it + /// scatters saturated noise across the field and hides the structure. Requiring a + /// handful leaves thin regions blank instead, which is honest. + /// + private static double?[] CellMedians( + ReadOnlySpan xValues, ReadOnlySpan yValues, ReadOnlySpan value, + double xLow, double xHigh, double yLow, double yHigh, int xBins, int yBins, int minimum) + { + var cells = new List[xBins * yBins]; + for (int i = 0; i < value.Length; i++) + { + int cx = Bucket(xValues[i], xLow, xHigh, xBins); + int cy = Bucket(yValues[i], yLow, yHigh, yBins); + if (cx < 0 || cy < 0) continue; + int index = (cy * xBins) + cx; + (cells[index] ??= new List()).Add(value[i]); + } + + var medians = new double?[cells.Length]; + for (int i = 0; i < cells.Length; i++) + { + List? rows = cells[i]; + if (rows is null || rows.Count < minimum) continue; + rows.Sort(); + medians[i] = rows[rows.Count / 2]; + } + + return medians; + } + + private static double CellScale(double?[] cells) + { + var magnitudes = new List(); + foreach (double? cell in cells) + { + if (cell is double v) magnitudes.Add(Math.Abs(v)); + } + + if (magnitudes.Count == 0) return 1; + magnitudes.Sort(); + + // The 97th percentile of what is drawn. Lower than this and the ramp saturates on + // ordinary cells, so the cell-to-cell noise in a median reads as structure; much + // higher and the field washes out again. + double limit = magnitudes[Math.Min(magnitudes.Count - 1, (int)(magnitudes.Count * 0.97))]; + return limit > 0 ? limit : 1; + } + + private static void DrawPanel( + Svg svg, double?[] cells, int xBins, int yBins, double scale, + double left, double top, double width, double height, + double rtMin, double rtMax, double mzMin, double mzMax, string title, bool showY) + { + var pixels = new byte[xBins * yBins * 3]; + FillBackground(pixels); + for (int i = 0; i < cells.Length; i++) + { + if (cells[i] is not double median) continue; + int cy = i / xBins, cx = i % xBins; + SetPixel(pixels, xBins, cx, yBins - 1 - cy, Diverging(median, scale)); + } + + svg.Image(left, top, width, height, Png.DataUri(pixels, xBins, yBins)); + svg.Rect(left, top, width, 1, Axis0); + svg.Rect(left, top + height, width, 1, Axis0); + svg.Rect(left, top, 1, height, Axis0); + svg.Rect(left + width, top, 1, height, Axis0); + svg.Text(left + (width / 2), top - 12, title, anchor: "middle", size: 16, bold: true); + + var x = new Axis(rtMin, rtMax, left, left + width); + foreach (double tick in x.Ticks(5)) + { + double px = x.Map(tick); + svg.Line(px, top + height, px, top + height + 4, Axis0, 1); + svg.Text(px, top + height + 16, x.Format(tick), anchor: "middle", size: 13, fill: Muted); + } + + svg.Text(left + (width / 2), top + height + 34, "retention time (min)", anchor: "middle", size: 15); + + if (!showY) return; + var y = new Axis(mzMin, mzMax, top, top + height, invert: true); + foreach (double tick in y.Ticks(5)) + { + double py = y.Map(tick); + svg.Line(left - 4, py, left, py, Axis0, 1); + svg.Text(left - 7, py + 3.5, y.Format(tick), anchor: "end", size: 13, fill: Muted); + } + } + + private static void VerticalColorBar( + Svg svg, double x, double top, double height, double scale, string unit) + { + const int steps = 48; + double band = height / steps; + for (int i = 0; i < steps; i++) + { + double t = 1 - (2.0 * i / (steps - 1)); + svg.Rect(x, top + (i * band), 13, band + 0.6, ColorOf(Diverging(t * scale, scale))); + } + + svg.Rect(x, top, 13, 1, Axis0); + svg.Rect(x, top + height, 13, 1, Axis0); + + string limit = scale.ToString(scale < 0.01 ? "0.####" : "0.###", CultureInfo.InvariantCulture); + svg.Text(x + 16, top + 8, "+" + limit, size: 10, fill: Muted); + svg.Text(x + 16, top + (height / 2) + 3, "0", size: 10, fill: Muted); + svg.Text(x + 16, top + height, "-" + limit, size: 10, fill: Muted); + svg.Text(x + 16, top + height + 15, unit, size: 10, fill: Muted); + } + + private static string ColorOf((byte R, byte G, byte B) c) => + "rgb(" + c.R.ToString(CultureInfo.InvariantCulture) + "," + + c.G.ToString(CultureInfo.InvariantCulture) + "," + + c.B.ToString(CultureInfo.InvariantCulture) + ")"; + + /// + /// Mass error against one feature as a density, before and after correction, side by + /// side. + /// + /// + /// + /// Colored by fragment count on a viridis ramp, which is what carries the information + /// here: a monochrome ramp has one usable dimension and spends most of it on pale + /// values, so the dense core and the sparse tail look much the same. Dark purple through + /// green to yellow separates two orders of magnitude legibly, and stays legible printed + /// in grayscale because it also rises monotonically in lightness. + /// + /// + /// Each panel is normalized to its own peak rather than to a shared one. Correcting + /// concentrates the distribution, so the after panel's peak is several times the before + /// panel's; on a shared scale the before panel would flatten to near-empty and the + /// structure that motivated the correction would disappear from the figure. + /// + /// + public static string FeatureVersusErrorPair( + ReadOnlySpan feature, ReadOnlySpan before, ReadOnlySpan after, + string featureName, string unit, int xBins = 72, int yBins = 56) + { + const int height = 400; + var svg = new Svg(Width, height); + if (feature.Length == 0) return Empty(svg, "No matched fragments."); + + bool paired = after.Length == before.Length; + double xLow = Percentile(feature, 0.002); + double xHigh = Percentile(feature, 0.998); + + // One vertical range for both panels: the after panel being visibly tighter is the + // result, and rescaling it away would hide exactly that. + double limit = SymmetricLimit(before, 0.99); + + const int top = 46; + const int bottom = 64; + const int gap = 24; + const int barWidth = 58; + int panelWidth = paired + ? (Width - Left - Right - gap - barWidth) / 2 + : Width - Left - Right - barWidth; + int panelHeight = height - top - bottom; + + int beforePeak = DrawDensityPanel( + svg, feature, before, xLow, xHigh, limit, xBins, yBins, + Left, top, panelWidth, panelHeight, paired ? "Before correction" : "As measured", + featureName, showY: true, unit: unit); + + int afterPeak = 0; + if (paired) + { + afterPeak = DrawDensityPanel( + svg, feature, after, xLow, xHigh, limit, xBins, yBins, + Left + panelWidth + gap, top, panelWidth, panelHeight, "After correction", + featureName, showY: false, unit: unit); + } + + ViridisBar(svg, Width - Right - barWidth + 10, top, panelHeight, beforePeak, afterPeak, paired); + return svg.ToString(); + } + + /// + /// Exponent for the density color ramp, as in matplotlib's PowerNorm. Below 1 it lifts + /// sparse cells; the further below, the more of the ramp goes to the sparse end. + /// + private const double DensityGamma = 0.4; + + /// Draws one density panel and returns the count in its busiest cell. + private static int DrawDensityPanel( + Svg svg, ReadOnlySpan feature, ReadOnlySpan error, + double xLow, double xHigh, double limit, int xBins, int yBins, + double left, double top, double width, double height, + string title, string featureName, bool showY, string unit) + { + var counts = new int[xBins * yBins]; + int peak = 0; + for (int i = 0; i < feature.Length; i++) + { + int cx = Bucket(feature[i], xLow, xHigh, xBins); + int cy = Bucket(error[i], -limit, limit, yBins); + if (cx < 0 || cy < 0) continue; + peak = Math.Max(peak, ++counts[(cy * xBins) + cx]); + } + + var pixels = new byte[xBins * yBins * 3]; + for (int i = 0; i < pixels.Length; i += 3) + { + // Empty stays white, as in the reference figures: an empty cell is an absence of + // data, not the low end of a density. + pixels[i] = 255; + pixels[i + 1] = 255; + pixels[i + 2] = 255; + } + + for (int i = 0; i < counts.Length; i++) + { + if (counts[i] == 0) continue; + + // A power law rather than a log or a straight fraction. Straight fraction leaves + // one bright cell in a dark field, because the core of a density like this runs + // orders of magnitude above its tails. A log overcorrects: with a peak of 2,854 it + // puts a 500-count cell at 0.78 of the ramp, so most of the core saturates to + // yellow and the structure inside it - which is the part worth looking at - is + // flattened. The exponent below puts that same cell at 0.50, keeping the top of + // the ramp for cells actually near the peak while still lifting the sparse tail + // clear of black. + double t = peak > 0 ? Math.Pow((double)counts[i] / peak, DensityGamma) : 0; + int cy = i / xBins, cx = i % xBins; + SetPixel(pixels, xBins, cx, yBins - 1 - cy, Viridis(t)); + } + + svg.Image(left, top, width, height, Png.DataUri(pixels, xBins, yBins)); + svg.Rect(left, top, width, 1, Axis0); + svg.Rect(left, top + height, width, 1, Axis0); + svg.Rect(left, top, 1, height, Axis0); + svg.Rect(left + width, top, 1, height, Axis0); + svg.Text(left + (width / 2), top - 12, title, anchor: "middle", size: 16, bold: true); + + var y = new Axis(-limit, limit, top, top + height, invert: true); + var x = new Axis(xLow, xHigh, left, left + width); + svg.Line(left, y.Map(0), left + width, y.Map(0), "#ffffff", 1.2, "4 3"); + MedianTrend(svg, feature, error, x, y, xLow, xHigh, xBins, left, left + width); + + foreach (double tick in x.Ticks(4)) + { + double px = x.Map(tick); + svg.Line(px, top + height, px, top + height + 4, Axis0, 1); + svg.Text(px, top + height + 16, x.Format(tick), anchor: "middle", size: 13, fill: Muted); + } + + svg.Text(left + (width / 2), top + height + 34, featureName, anchor: "middle", size: 15); + + if (showY) + { + foreach (double tick in y.Ticks(5)) + { + double py = y.Map(tick); + svg.Line(left - 4, py, left, py, Axis0, 1); + svg.Text(left - 7, py + 4, y.Format(tick), anchor: "end", size: 13, fill: Muted); + } + + svg.Text(13, top + (height / 2), $"mass error ({unit})", anchor: "middle", size: 15, rotate: -90); + } + + return peak; + } + + /// + /// Median error per column, drawn over the density. + /// + /// + /// Cased - a dark stroke under a white one - because viridis runs from near-black to + /// near-yellow and no single color reads against both ends of it. + /// + private static void MedianTrend( + Svg svg, ReadOnlySpan feature, ReadOnlySpan error, + Axis x, Axis y, double low, double high, int bins, double clipLeft, double clipRight) + { + var buckets = new List[bins]; + for (int i = 0; i < feature.Length; i++) + { + int b = Bucket(feature[i], low, high, bins); + if (b < 0) continue; + (buckets[b] ??= new List()).Add(error[i]); + } + + var points = new List<(double X, double Y)>(); + double step = (high - low) / bins; + for (int b = 0; b < bins; b++) + { + List? values = buckets[b]; + // A column of a handful of rows is noise, and a trend line through noise reads as + // signal. + if (values is null || values.Count < 20) continue; + values.Sort(); + double px = x.Map(low + ((b + 0.5) * step)); + if (px < clipLeft || px > clipRight) continue; + points.Add((px, y.Map(values[values.Count / 2]))); + } + + var span = System.Runtime.InteropServices.CollectionsMarshal.AsSpan(points); + svg.Polyline(span, "#1a1d21", 3.2); + svg.Polyline(span, "#ffffff", 1.6); + } + + private static void ViridisBar( + Svg svg, double x, double top, double height, int beforePeak, int afterPeak, bool paired) + { + const int steps = 48; + double band = height / steps; + for (int i = 0; i < steps; i++) + { + double t = 1 - ((double)i / (steps - 1)); + svg.Rect(x, top + (i * band), 13, band + 0.6, ColorOf(Viridis(t))); + } + + svg.Rect(x, top, 13, 1, Axis0); + svg.Rect(x, top + height, 13, 1, Axis0); + svg.Text(x + 16, top + 8, "peak", size: 10, fill: Muted); + svg.Text(x + 16, top + height, "0", size: 10, fill: Muted); + svg.Text(x + 16, top + height + 15, "fragments", size: 10, fill: Muted); + + string peaks = paired + ? $"peak {beforePeak:N0} / {afterPeak:N0}" + : $"peak {beforePeak:N0}"; + svg.Text(x + 16, top + height + 29, peaks, size: 9, fill: Muted); + } + + /// + /// The viridis ramp, interpolated between its usual anchors. + /// + /// + /// Chosen for the reason it is usually chosen: it is perceptually near-uniform, so equal + /// steps in density look like equal steps in color, and it rises monotonically in + /// lightness so it survives being printed in grayscale. + /// + private static (byte R, byte G, byte B) Viridis(double t) + { + (double R, double G, double B)[] anchors = + { + (68, 1, 84), (72, 40, 120), (62, 74, 137), (49, 104, 142), (38, 130, 142), + (31, 158, 137), (53, 183, 121), (109, 205, 89), (180, 222, 44), (253, 231, 37), + }; + + double clamped = Math.Clamp(t, 0, 1) * (anchors.Length - 1); + int i = Math.Min((int)clamped, anchors.Length - 2); + double f = clamped - i; + + return ( + (byte)Math.Round(anchors[i].R + ((anchors[i + 1].R - anchors[i].R) * f)), + (byte)Math.Round(anchors[i].G + ((anchors[i + 1].G - anchors[i].G) * f)), + (byte)Math.Round(anchors[i].B + ((anchors[i + 1].B - anchors[i].B) * f))); + } + + /// + /// Each fold's accuracy against the pooled figure, with a band at one standard + /// deviation either side of the fold mean. + /// + /// + /// The variance is the point of plotting this at all. A single held-out number says how + /// the model did on one split; the picture says whether that number was luck. Folds + /// sitting almost on top of each other mean the estimate is stable and the headline + /// figure can be quoted as-is. Folds scattered across the band mean the cohort contains + /// regions the model handles very differently, and the pooled number is an average over + /// them rather than a description of any of them. + /// + public static string FoldSpread( + IReadOnlyList perFold, double pooled, double spread, string unit, string metric) + { + const int height = 215; + var svg = new Svg(Width, height); + if (perFold.Count == 0) return Empty(svg, "No folds to show."); + + double lowest = pooled, highest = pooled; + foreach (double value in perFold) + { + lowest = Math.Min(lowest, value); + highest = Math.Max(highest, value); + } + + // Pad by the spread so the band is visible even when the folds are nearly identical, + // which is the common and desirable case. + double pad = Math.Max(double.IsNaN(spread) ? 0 : spread * 2, (highest - lowest) * 0.6); + if (pad <= 0) pad = Math.Abs(pooled) * 0.02; + if (pad <= 0) pad = 1; + + var x = new Axis(lowest - pad, highest + pad, Left, Width - Right); + double axisY = height - 54; + double dotY = axisY - 46; + + if (!double.IsNaN(spread) && spread > 0) + { + double mean = 0; + foreach (double value in perFold) mean += value; + mean /= perFold.Count; + + double bandLow = x.Map(mean - spread); + double bandHigh = x.Map(mean + spread); + svg.Rect(bandLow, dotY - 22, bandHigh - bandLow, 44, After, + "fill-opacity=\"0.12\""); + svg.Text((bandLow + bandHigh) / 2, dotY - 28, "+/- 1 sd", anchor: "middle", + size: 10, fill: Muted); + } + + double pooledX = x.Map(pooled); + svg.Line(pooledX, dotY - 26, pooledX, dotY + 26, Before, 2); + svg.Text(pooledX, dotY + 40, "pooled " + pooled.ToString("0.0000", CultureInfo.InvariantCulture), + anchor: "middle", size: 10, fill: Before); + + for (int i = 0; i < perFold.Count; i++) + { + double px = x.Map(perFold[i]); + // Spread the dots vertically so folds with near-identical values stay countable + // rather than drawing on top of one another. + double py = dotY - 14 + (i * (28.0 / Math.Max(1, perFold.Count - 1))); + svg.Rect(px - 3, py - 3, 6, 6, After); + svg.Text(px + 7, py + 3.5, (i + 1).ToString(CultureInfo.InvariantCulture), size: 9, fill: Muted); + } + + foreach (double tick in x.Ticks(6)) + { + double px = x.Map(tick); + svg.Line(px, axisY, px, axisY + 4, Axis0, 1); + svg.Text(px, axisY + 18, x.Format(tick), anchor: "middle", size: 13, fill: Muted); + } + + svg.Line(Left, axisY, Width - Right, axisY, Axis0, 1); + svg.Text(Left, 22, $"{metric} per fold ({unit})", size: 16, bold: true); + return svg.ToString(); + } + + public static string FeatureImportance(IReadOnlyList names, IReadOnlyList importance) + { + int rows = Math.Min(names.Count, importance.Count); + int height = Math.Max(190, 44 + (rows * 26)); + var svg = new Svg(Width, height); + if (rows == 0) return Empty(svg, "Importance was not computed."); + + var order = new int[rows]; + for (int i = 0; i < rows; i++) order[i] = i; + Array.Sort(order, (a, b) => importance[b].CompareTo(importance[a])); + + double max = 0; + foreach (double value in importance) max = Math.Max(max, value); + if (max <= 0) max = 1; + + const int labelWidth = 230; + double barLeft = labelWidth + 10; + double barSpan = Width - barLeft - 60; + + for (int i = 0; i < rows; i++) + { + int index = order[i]; + double y = 28 + (i * 26); + svg.Text(labelWidth, y + 13, names[index], anchor: "end", size: 13); + double barWidth = barSpan * (importance[index] / max); + svg.Rect(barLeft, y + 2, barWidth, 14, After); + svg.Text( + barLeft + barWidth + 6, y + 13, + importance[index].ToString("0.000", CultureInfo.InvariantCulture), + size: 11, fill: Muted); + } + + return svg.ToString(); + } + + // ---- helpers --------------------------------------------------------------------- + + private static string Empty(Svg svg, string message) + { + svg.Text(svg.Width / 2.0, svg.Height / 2.0, message, anchor: "middle", fill: Muted); + return svg.ToString(); + } + + private static void Frame(Svg svg, Axis x, Axis y, string xLabel, string yLabel, bool drawGrid = true) + { + foreach (double tick in y.Ticks(6)) + { + double py = y.Map(tick); + if (drawGrid) svg.Line(Left, py, Width - Right, py, Grid, 1); + svg.Text(Left - 8, py + 4, y.Format(tick), anchor: "end", size: 13, fill: Muted); + } + + foreach (double tick in x.Ticks(7)) + { + double px = x.Map(tick); + if (drawGrid) svg.Line(px, Top, px, Height - Bottom, Grid, 1); + svg.Text(px, Height - Bottom + 17, x.Format(tick), anchor: "middle", size: 13, fill: Muted); + } + + svg.Line(Left, Height - Bottom, Width - Right, Height - Bottom, Axis0, 1); + svg.Line(Left, Top, Left, Height - Bottom, Axis0, 1); + svg.Text(Left + ((Width - Right - Left) / 2.0), Height - 6, xLabel, anchor: "middle", size: 15); + svg.Text(14, Top + ((Height - Bottom - Top) / 2.0), yLabel, anchor: "middle", size: 15, rotate: -90); + } + + private static void DrawBars( + Svg svg, int[] counts, Axis x, Axis y, double low, double high, string fill, double opacity) + { + double step = (high - low) / counts.Length; + double baseline = y.Map(0); + for (int i = 0; i < counts.Length; i++) + { + if (counts[i] == 0) continue; + double x0 = x.Map(low + (i * step)); + double x1 = x.Map(low + ((i + 1) * step)); + double top = y.Map(counts[i]); + svg.Rect(x0, top, Math.Max(0.6, x1 - x0), baseline - top, fill, + $"fill-opacity=\"{opacity.ToString(CultureInfo.InvariantCulture)}\""); + } + } + + private static void MedianTrend( + Svg svg, ReadOnlySpan feature, ReadOnlySpan error, + Axis x, Axis y, double low, double high, int bins, string stroke) + { + var buckets = new List[bins]; + for (int i = 0; i < feature.Length; i++) + { + int b = Bucket(feature[i], low, high, bins); + if (b < 0) continue; + (buckets[b] ??= new List()).Add(error[i]); + } + + var points = new List<(double X, double Y)>(); + double step = (high - low) / bins; + for (int b = 0; b < bins; b++) + { + List? values = buckets[b]; + // A bucket with a handful of rows is noise, and a trend line drawn through noise + // reads as signal. Require enough to make the median mean something. + if (values is null || values.Count < 20) continue; + values.Sort(); + points.Add((x.Map(low + ((b + 0.5) * step)), y.Map(values[values.Count / 2]))); + } + + svg.Polyline(System.Runtime.InteropServices.CollectionsMarshal.AsSpan(points), stroke, 2); + } + + private static void Legend(Svg svg, bool hasAfter, string beforeLabel = "before", string afterLabel = "after") + { + double x = Left + 4; + svg.Rect(x, 13, 11, 11, Before); + svg.Text(x + 17, 24, beforeLabel, size: 13); + if (!hasAfter) return; + svg.Rect(x + 110, 13, 11, 11, After); + svg.Text(x + 130, 24, afterLabel, size: 13); + } + + private static void ColorBar(Svg svg, double scale, string unit) + { + const int steps = 40; + double barWidth = 150.0; + double x0 = Width - Right - barWidth; + for (int i = 0; i < steps; i++) + { + double t = ((i / (double)(steps - 1)) * 2) - 1; + (byte r, byte g, byte b) = Diverging(t * scale, scale); + svg.Rect(x0 + (i * (barWidth / steps)), 10, (barWidth / steps) + 0.5, 9, $"rgb({r},{g},{b})"); + } + + svg.Text(x0 - 6, 18, $"-{scale.ToString("0.###", CultureInfo.InvariantCulture)}", anchor: "end", size: 9, fill: Muted); + svg.Text(Width - Right + 2, 18, $"+{scale.ToString("0.###", CultureInfo.InvariantCulture)} {unit}", anchor: "start", size: 9, fill: Muted); + } + + /// Places a density buffer in the plot area, scaled to fill it. + private static void Raster(Svg svg, byte[] pixels, int xBins, int yBins) => + svg.Image(Left, Top, Width - Right - Left, Height - Bottom - Top, Png.DataUri(pixels, xBins, yBins)); + + /// + /// Fills the buffer with the panel background, so an empty cell is not black. + /// + /// + /// A near-white constant rather than the theme's background variable: this is a raster, + /// so it cannot follow the reader's color scheme the way the vector layers do. The + /// density ramps run light-to-dark, which reads correctly on either theme. + /// + private static void FillBackground(byte[] pixels) + { + for (int i = 0; i < pixels.Length; i += 3) + { + pixels[i] = 252; + pixels[i + 1] = 252; + pixels[i + 2] = 253; + } + } + + private static void SetPixel(byte[] pixels, int width, int x, int y, (byte R, byte G, byte B) color) + { + int offset = ((y * width) + x) * 3; + pixels[offset] = color.R; + pixels[offset + 1] = color.G; + pixels[offset + 2] = color.B; + } + + /// Blue-white-red, so the sign of the error is readable and zero is blank. + private static (byte R, byte G, byte B) Diverging(double value, double scale) + { + // Quantized to 32 steps, finer than the eye resolves on a diverging ramp, and a + // limited palette is what lets deflate compress the panel down to a few kilobytes. + double t = Math.Round(Math.Clamp(value / scale, -1, 1) * 32) / 32; + if (t < 0) + { + double k = -t; + return ((byte)(255 - (k * 190)), (byte)(255 - (k * 130)), 255); + } + + return (255, (byte)(255 - (t * 150)), (byte)(255 - (t * 165))); + } + + private static (byte R, byte G, byte B) Density(double intensity) + { + double t = Math.Round(Math.Clamp(intensity, 0, 1) * 24) / 24; + return ((byte)(238 - (t * 200)), (byte)(242 - (t * 150)), (byte)(248 - (t * 60))); + } + + private static int Bucket(double value, double low, double high, int bins) + { + if (double.IsNaN(value) || value < low || value > high) return -1; + int b = (int)((value - low) / (high - low) * bins); + return Math.Clamp(b, 0, bins - 1); + } + + private static int[] Bin(ReadOnlySpan values, double low, double high, int bins) + { + var counts = new int[bins]; + foreach (double value in values) + { + int b = Bucket(value, low, high, bins); + if (b >= 0) counts[b]++; + } + + return counts; + } + + private static (double Min, double Max) Range(ReadOnlySpan values) + { + double min = double.PositiveInfinity, max = double.NegativeInfinity; + foreach (double value in values) + { + if (double.IsNaN(value)) continue; + if (value < min) min = value; + if (value > max) max = value; + } + + return double.IsInfinity(min) ? (0, 1) : (min, max); + } + + private static double SymmetricLimit(ReadOnlySpan values, double quantile) + { + double limit = Math.Max( + Math.Abs(Percentile(values, 1 - quantile)), + Math.Abs(Percentile(values, quantile))); + return limit > 0 ? limit : 1; + } + + private static double Percentile(ReadOnlySpan values, double quantile) + { + if (values.Length == 0) return 0; + + // Sampled rather than sorting nine million rows to place an axis limit. The stride + // is deterministic, so the same input always yields the same figure. + const int cap = 20000; + int stride = Math.Max(1, values.Length / cap); + var sample = new List(Math.Min(cap + 1, values.Length)); + for (int i = 0; i < values.Length; i += stride) + { + if (!double.IsNaN(values[i])) sample.Add(values[i]); + } + + if (sample.Count == 0) return 0; + sample.Sort(); + int index = Math.Clamp((int)(quantile * (sample.Count - 1)), 0, sample.Count - 1); + return sample[index]; + } +} diff --git a/dotnet/MARS/Report/ErrorScale.cs b/dotnet/MARS/Report/ErrorScale.cs new file mode 100644 index 0000000..17dc97b --- /dev/null +++ b/dotnet/MARS/Report/ErrorScale.cs @@ -0,0 +1,80 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Globalization; +using MARS.Core; + +namespace MARS.Report; + +/// +/// The scale a QC report expresses mass error in, and how to render a number on it. +/// +/// +/// Which one is right is a property of the instrument. A trap's error is roughly constant in +/// Th, so Th is the scale on which its figures are flat across the m/z range; a +/// high-resolution analyzer's error is roughly constant in ppm, and drawing it in Th produces +/// a fan that widens with m/z out of nothing but the choice of units. Four decimal places +/// suits Th, where the interesting digits are hundredths; two suit ppm, where they are ones. +/// +public sealed class ErrorScale +{ + public static readonly ErrorScale Th = new("Th", "0.0000"); + + public static readonly ErrorScale Ppm = new("ppm", "0.00"); + + private ErrorScale(string unit, string numberFormat) + { + Unit = unit; + NumberFormat = numberFormat; + } + + public string Unit { get; } + + private string NumberFormat { get; } + + public bool IsPpm => ReferenceEquals(this, Ppm); + + public string Format(double value) => value.ToString(NumberFormat, CultureInfo.InvariantCulture); + + public string FormatSigned(double value) => + value.ToString("+" + NumberFormat + ";-" + NumberFormat, CultureInfo.InvariantCulture); + + /// + /// Converts per-row error to this scale, using each row's own m/z rather than an average + /// - the fragments in one run span a wide enough m/z range that a single divisor would be + /// wrong at both ends. + /// + public double[] Convert(double[] error, double[] mz) + { + // An empty error array is not a mismatch: `mars qc` draws the report with no + // after-correction series, and passes one. + if (!IsPpm || error.Length == 0) return error; + + // Beyond that the two arrays have to describe the same rows, because each is converted + // by its own m/z. Filling a short one with zeros would put 0 ppm into a QC figure for + // every row past the end of it - which reads as a perfectly calibrated fragment. + if (error.Length != mz.Length) + { + throw new ArgumentException( + $"{error.Length:N0} error values and {mz.Length:N0} m/z values: per-row ppm " + + "conversion needs one m/z per error.", + nameof(mz)); + } + + var converted = new double[error.Length]; + for (int i = 0; i < error.Length; i++) + { + // A non-positive m/z cannot be converted and is left at zero rather than made + // infinite; no real fragment has one. + converted[i] = mz[i] > 0 ? error[i] / mz[i] * 1e6 : 0; + } + + return converted; + } + + /// Picks whichever of the two summaries is on this scale. + public ErrorSummary? Pick(ErrorSummary? th, ErrorSummary? ppm) => IsPpm ? ppm ?? th : th; + + /// Picks whichever of the two fold measurements is on this scale. + public FoldMetrics Pick(FoldMetrics th, FoldMetrics? ppm) => IsPpm && ppm is FoldMetrics p ? p : th; +} diff --git a/dotnet/MARS/Report/Png.cs b/dotnet/MARS/Report/Png.cs new file mode 100644 index 0000000..cebb5ed --- /dev/null +++ b/dotnet/MARS/Report/Png.cs @@ -0,0 +1,125 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Minimal PNG encoder for the QC report's density layers. + +using System; +using System.Buffers.Binary; +using System.IO; +using System.IO.Compression; + +namespace MARS.Report; + +/// +/// Encodes an RGB pixel buffer as a PNG, for embedding in the report as a data URI. +/// +/// The density panels are the report's bulk: a couple of dozen panels of several thousand +/// cells each. Drawing them as SVG rectangles produced a six-megabyte file, too large to +/// email, and merging runs only reached five. As a PNG the same panel is a few kilobytes, +/// because a run of similar colors is exactly what deflate is good at. +/// +/// Axes, labels and trend lines stay vector; only the density itself is raster, which is +/// also what a plotting library would do. There is no imaging dependency here - a PNG is a +/// header, a zlib stream and a CRC, and .NET has the zlib. +/// +public static class Png +{ + private static readonly byte[] Signature = { 137, 80, 78, 71, 13, 10, 26, 10 }; + + private static readonly uint[] CrcTable = BuildCrcTable(); + + /// Row-major RGB triples, * * 3 bytes. + public static byte[] Encode(byte[] rgb, int width, int height) + { + if (width <= 0 || height <= 0) + { + throw new ArgumentOutOfRangeException( + width <= 0 ? nameof(width) : nameof(height), + $"A PNG needs positive dimensions; got {width}x{height}."); + } + + // In long arithmetic: a large enough width times height overflows int and can wrap to + // the buffer's actual length, letting a mismatched buffer through to be written as a + // malformed file. + long expected = (long)width * height * 3; + if (rgb.Length != expected) + { + throw new ArgumentException( + $"Pixel buffer is {rgb.Length:N0} bytes; {width}x{height} RGB needs {expected:N0}.", + nameof(rgb)); + } + + using var png = new MemoryStream(); + png.Write(Signature, 0, Signature.Length); + + var header = new byte[13]; + BinaryPrimitives.WriteInt32BigEndian(header.AsSpan(0), width); + BinaryPrimitives.WriteInt32BigEndian(header.AsSpan(4), height); + header[8] = 8; // bits per channel + header[9] = 2; // color type 2: truecolor RGB + header[10] = 0; // deflate + header[11] = 0; // adaptive filtering + header[12] = 0; // no interlace + WriteChunk(png, "IHDR", header); + + // Each scanline is prefixed with its filter type. Filter 0 (none) keeps this simple; + // the color ramps are quantized, so deflate already finds long runs. + var raw = new byte[height * ((width * 3) + 1)]; + int stride = (width * 3) + 1; + for (int y = 0; y < height; y++) + { + raw[y * stride] = 0; + Buffer.BlockCopy(rgb, y * width * 3, raw, (y * stride) + 1, width * 3); + } + + using var compressed = new MemoryStream(); + using (var deflate = new ZLibStream(compressed, CompressionLevel.SmallestSize, leaveOpen: true)) + deflate.Write(raw, 0, raw.Length); + + WriteChunk(png, "IDAT", compressed.ToArray()); + WriteChunk(png, "IEND", Array.Empty()); + return png.ToArray(); + } + + /// Encodes to a data: URI, ready to drop into an img or SVG image element. + public static string DataUri(byte[] rgb, int width, int height) => + "data:image/png;base64," + Convert.ToBase64String(Encode(rgb, width, height)); + + private static void WriteChunk(Stream stream, string type, byte[] data) + { + Span length = stackalloc byte[4]; + BinaryPrimitives.WriteInt32BigEndian(length, data.Length); + stream.Write(length); + + var typeBytes = new byte[4]; + for (int i = 0; i < 4; i++) typeBytes[i] = (byte)type[i]; + stream.Write(typeBytes, 0, 4); + stream.Write(data, 0, data.Length); + + // The CRC covers the type and the data, but not the length. + uint crc = 0xFFFFFFFF; + crc = Crc(crc, typeBytes); + crc = Crc(crc, data); + Span checksum = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(checksum, crc ^ 0xFFFFFFFF); + stream.Write(checksum); + } + + private static uint Crc(uint crc, byte[] data) + { + foreach (byte b in data) crc = CrcTable[(crc ^ b) & 0xFF] ^ (crc >> 8); + return crc; + } + + private static uint[] BuildCrcTable() + { + var table = new uint[256]; + for (uint n = 0; n < 256; n++) + { + uint c = n; + for (int k = 0; k < 8; k++) + c = (c & 1) != 0 ? 0xEDB88320 ^ (c >> 1) : c >> 1; + table[n] = c; + } + + return table; + } +} diff --git a/dotnet/MARS/Report/QcHtmlReport.cs b/dotnet/MARS/Report/QcHtmlReport.cs new file mode 100644 index 0000000..0b5c435 --- /dev/null +++ b/dotnet/MARS/Report/QcHtmlReport.cs @@ -0,0 +1,524 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// The single-file HTML QC report. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using MARS.Core; + +namespace MARS.Report; + +/// +/// Writes the QC figures and summary as one self-contained HTML file. +/// +/// Self-contained is the requirement, not a nicety: the report is meant to be attached to +/// an email and opened by someone who does not have the data, the tool, or a network path +/// back to either. Everything is inline - the figures are SVG elements in the markup, the +/// styling is one embedded stylesheet, and there is no script. Nothing is fetched when the +/// file is opened, which also means it renders in mail clients that block remote content. +/// +public static class QcHtmlReport +{ + /// Per-row data the figures are drawn from. + public sealed class Data + { + public required double[] ErrorBefore { get; init; } + + /// Error left after the model's correction. Empty when no model was fitted. + public required double[] ErrorAfter { get; init; } + + public required double[] RetentionTime { get; init; } + + public required double[] FragmentMz { get; init; } + + /// Feature name to per-row values, in model order. + public required IReadOnlyList<(string Name, double[] Values)> Features { get; init; } + + public required IReadOnlyList ImportanceNames { get; init; } + + public required IReadOnlyList Importance { get; init; } + + /// Cross-validation results, or null when a single model was fitted. + public CrossValidationReport? CrossValidation { get; init; } + } + + /// Training statistics, or null when no model was fitted. + /// + /// Summary of the uncorrected error. Used when is null, + /// which is the `mars qc` case: there is no before-and-after to show, but the error + /// that is there is the whole point of the report. + /// + public static void Write( + string path, + Data data, + TrainingStatistics? statistics, + MatchStatistics matchStatistics, + IReadOnlyList inputFiles, + string toleranceDescription, + string version, + ErrorSummary? uncorrected = null, + ErrorScale? scale = null) + { + scale ??= ErrorScale.Th; + + // The charts are drawn from per-row values, so on a ppm report the rows themselves are + // converted once here and every chart below is already on the right scale. + double[] errorBefore = scale.Convert(data.ErrorBefore, data.FragmentMz); + double[] errorAfter = scale.Convert(data.ErrorAfter, data.FragmentMz); + + string? directory = Path.GetDirectoryName(Path.GetFullPath(path)); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + + var html = new StringBuilder(1 << 20); + html.Append(""); + html.Append(""); + html.Append("MARS QC report"); + html.Append("
"); + + html.Append("

MARS QC report

"); + if (statistics is null) + html.Append("

Pre-calibration. No model was fitted.

"); + html.Append("

MARS ").Append(Svg.Escape(version)).Append(" · ") + .Append(inputFiles.Count.ToString("N0", CultureInfo.InvariantCulture)) + .Append(inputFiles.Count == 1 ? " input file" : " input files").Append("

"); + + AppendVerdict(html, statistics, uncorrected, data.CrossValidation, scale); + AppendSummaryTables( + html, statistics, uncorrected, matchStatistics, inputFiles, toleranceDescription, scale); + + bool corrected = data.ErrorAfter.Length == data.ErrorBefore.Length; + + Figure(html, "Mass error distribution", + corrected + ? "The uncorrected error against what is left in these files after correction. " + + "If these two distributions are not visibly different, the model found " + + "nothing to remove." + : "The mass error as measured, before any correction. Its width is what a model " + + "would have to work with; how much of it is removable is what calibrating " + + "would show.", + Charts.ErrorHistogram(errorBefore, errorAfter, scale.Unit)); + + Figure(html, "Error across retention time and fragment m/z", + corrected + ? "Median error per cell, both panels on one color scale so they can be compared " + + "directly. Structure on the left is the systematic component MARS exists to " + + "remove; the right panel washing out is the goal. Blank cells held too few " + + "fragments to take a median from." + : "Median error per cell. Visible structure - bands, gradients, blocks - is " + + "systematic error, and systematic error is the kind MARS can remove. A " + + "featureless panel means the error is mostly noise. Blank cells held too few " + + "fragments to take a median from.", + Charts.ErrorHeatmapPair( + data.RetentionTime, data.FragmentMz, errorBefore, errorAfter, scale.Unit)); + + AppendCrossValidation(html, data.CrossValidation, scale); + + if (data.Importance.Count > 0) + { + Figure(html, "Feature importance", + "Permutation importance: how much the validation error degrades when one feature " + + "is shuffled. A feature near zero is carrying no weight and could be dropped.", + Charts.FeatureImportance(PrettyNames(data.ImportanceNames), data.Importance)); + } + + if (data.Features.Count > 0) + { + html.Append("

Error against each feature

"); + html.Append("

Fragment count per cell, with the " + + "median error per column drawn over it" + + (corrected ? ", before and after correction side by side" : string.Empty) + + ". The trend line is the part to read: a sloped line is a real dependence" + + (corrected + ? ", and a flat line in the right panel means the model captured it. " + + "Each panel is scaled to its own busiest cell, since correcting " + + "concentrates the distribution." + : " that a model could exploit; a flat line means this feature says " + + "nothing about the error here.") + + "

"); + } + + foreach ((string name, double[] values) in data.Features) + { + Figure(html, Pretty(name), null, + Charts.FeatureVersusErrorPair( + values, errorBefore, errorAfter, Pretty(name), scale.Unit)); + } + + html.Append("
"); + File.WriteAllText(path, html.ToString(), new UTF8Encoding(false)); + } + + private static void AppendVerdict( + StringBuilder html, TrainingStatistics? statistics, ErrorSummary? uncorrected, + CrossValidationReport? crossValidation, ErrorScale scale) + { + if (statistics is null) + { + AppendPreCalibrationVerdict(html, uncorrected, scale); + return; + } + + double before = (scale.Pick(statistics.Before, statistics.BeforePpm) ?? statistics.Before).Mad; + double after = (scale.Pick(statistics.After, statistics.AfterPpm) ?? statistics.After).Mad; + if (!(before > 0)) return; + + double reduction = 100 * (1 - (after / before)); + // Say what the numbers mean rather than leaving the reader to decide what counts as + // a good result. A run with little systematic error left is a legitimate outcome and + // should not read as a failure. + string verdict = reduction switch + { + >= 25 => "The correction removed a substantial part of the mass error.", + >= 10 => "The correction removed a modest part of the mass error.", + >= 2 => "The correction changed little. This data was already close to calibrated.", + _ => "The correction removed essentially nothing. There is no systematic error here " + + "to remove, and the corrected file is little different from the input.", + }; + + html.Append("
") + .Append(reduction.ToString("0.0", CultureInfo.InvariantCulture)) + .Append("% reduction in median absolute error, ") + .Append(scale.Format(before)).Append(" → ") + .Append(scale.Format(after)).Append(' ').Append(scale.Unit).Append(". ") + .Append(verdict); + + if (crossValidation is CrossValidationReport cv) + { + // Two numbers, two questions. The one above is what these files now look like; + // this is what the same procedure achieves on a run it was not fitted to. + FoldMetrics outOfFold = scale.Pick(cv.OutOfFold, cv.OutOfFoldPpm); + html.Append(" On data not used to fit, cross-validation puts it at ") + .Append(scale.Format(outOfFold.Mad)).Append(' ').Append(scale.Unit).Append(" (") + .Append(outOfFold.MadReduction.ToString("0.0", CultureInfo.InvariantCulture)) + .Append("%)."); + } + + html.Append("
"); + } + + /// + /// What `mars qc` can honestly say: how big the error is, and how much of it is a plain + /// offset. It cannot say how much is removable - only fitting a model answers that - so + /// it does not pretend to. + /// + private static void AppendPreCalibrationVerdict( + StringBuilder html, ErrorSummary? uncorrected, ErrorScale scale) + { + if (uncorrected is not ErrorSummary summary || summary.Count == 0) return; + + html.Append("
") + .Append(scale.Format(summary.Mad)).Append(' ').Append(scale.Unit) + .Append(" median absolute error across ") + .Append(summary.Count.ToString("N0", CultureInfo.InvariantCulture)) + .Append(" matched fragments, with a median of ") + .Append(scale.Format(summary.Median)).Append(' ').Append(scale.Unit).Append(". "); + + // A median well away from zero is a straight offset across the whole run, which is + // the most obviously correctable thing there is. + double bias = Math.Abs(summary.Median); + html.Append(bias > summary.Mad * 0.5 + ? "The median is a long way from zero, so a systematic offset runs through the " + + "whole cohort. That part is straightforwardly correctable." + : "The median is close to zero, so there is no large constant offset. Whether the " + + "spread is systematic enough to remove is what fitting a model would show."); + + html.Append(" Run mars calibrate to find out how much of this is " + + "removable.
"); + } + + /// + /// Per-fold accuracy and the spread across folds. + /// + /// + /// The spread is the point. One held-out number says how the model did on one split; + /// five say whether that number was luck. A tight spread means the estimate is stable, + /// a wide one means the cohort has regions the model handles very differently and the + /// headline figure is an average over them. + /// + private static void AppendCrossValidation( + StringBuilder html, CrossValidationReport? cv, ErrorScale scale) + { + if (cv is null) return; + + FoldMetrics[] perFold = cv.PerFold; + if (scale.IsPpm && cv.PerFoldPpm is FoldMetrics[] ppmFolds) perFold = ppmFolds; + FoldMetrics pooled = scale.Pick(cv.OutOfFold, cv.OutOfFoldPpm); + FoldMetrics inSample = scale.Pick(cv.InSample, cv.InSamplePpm); + + html.Append("

Cross-validation

"); + html.Append("

") + .Append(cv.Folds.ToString(CultureInfo.InvariantCulture)) + .Append(" folds split by peptide, over ") + .Append(cv.Groups.ToString("N0", CultureInfo.InvariantCulture)) + .Append(" peptides. Every row was scored by a model that never saw its peptide, so " + + "so these figures estimate what this correction would achieve on a run it was " + + "not fitted to, which is what mars apply does. The figures elsewhere describe " + + "these files, which the applied model was fitted to - as mass calibration " + + "normally is.

"); + + html.Append("
"); + html.Append("" + + ""); + + for (int i = 0; i < perFold.Length; i++) + { + FoldMetrics fold = perFold[i]; + html.Append("") + .Append("") + .Append("") + .Append("") + .Append("") + .Append(""); + } + + html.Append("") + .Append("") + .Append("") + .Append("") + .Append("") + .Append(""); + + html.Append("") + .Append("") + .Append("") + .Append("") + .Append(""); + + html.Append("
foldrowsMAD (") + .Append(scale.Unit).Append(")RMS (").Append(scale.Unit) + .Append(")reductionPearson r
").Append(i + 1).Append("").Append(fold.Rows.ToString("N0", CultureInfo.InvariantCulture)).Append("").Append(scale.Format(fold.Mad)).Append("").Append(scale.Format(fold.Rms)).Append("").Append(fold.MadReduction.ToString("0.0", CultureInfo.InvariantCulture)).Append("%").Append(Format(fold.PearsonR)).Append("
pooled").Append(pooled.Rows.ToString("N0", CultureInfo.InvariantCulture)).Append("").Append(scale.Format(pooled.Mad)).Append("").Append(scale.Format(pooled.Rms)).Append("").Append(pooled.MadReduction.ToString("0.0", CultureInfo.InvariantCulture)).Append("%").Append(Format(pooled.PearsonR)).Append("
spread") + .Append(PlusMinus(CrossValidationReport.Spread(perFold, static f => f.Mad), scale)).Append("") + .Append(PlusMinus(CrossValidationReport.Spread(perFold, static f => f.Rms), scale)).Append("").Append(PlusMinus(cv.MadReductionSpread)).Append("").Append(PlusMinus(cv.PearsonRSpread)).Append("
Spread is the standard deviation across folds.
"); + html.Append("
"); + + var foldMad = new double[perFold.Length]; + var foldR = new double[perFold.Length]; + for (int i = 0; i < perFold.Length; i++) + { + foldMad[i] = perFold[i].Mad; + foldR[i] = perFold[i].PearsonR; + } + + Figure(html, null, + "Each fold's accuracy against the pooled figure. Folds sitting close together mean " + + "the estimate is stable and the pooled number can be read as-is; folds scattered " + + "across the band mean the cohort has regions the model handles very differently, " + + "and the pooled number is an average over them.", + Charts.FoldSpread( + foldMad, pooled.Mad, CrossValidationReport.Spread(perFold, static f => f.Mad), scale.Unit, + "Median absolute residual")); + + Figure(html, null, null, + Charts.FoldSpread(foldR, pooled.PearsonR, cv.PearsonRSpread, "r", "Pearson correlation")); + + html.Append("
Gap ") + .Append(scale.Format(pooled.Mad - inSample.Mad)).Append(' ').Append(scale.Unit) + .Append(". The correction leaves ") + .Append(scale.Format(inSample.Mad)).Append(' ').Append(scale.Unit) + .Append(" on the data it was fitted to and ") + .Append(scale.Format(pooled.Mad)).Append(' ').Append(scale.Unit) + .Append(" on peptides it had not seen. ") + .Append(OptimismVerdict(pooled.Mad - inSample.Mad, pooled.Mad)) + .Append("
"); + } + + private static string PlusMinus(double value, ErrorScale scale) => + double.IsNaN(value) ? "n/a" : "+/-" + scale.Format(value); + + private static string PlusMinus(double value) => + double.IsNaN(value) ? "n/a" : "+/-" + value.ToString("0.0000", CultureInfo.InvariantCulture); + + /// Out-of-fold minus in-sample, on the report's scale. + /// Out-of-fold MAD on the same scale, to judge the gap against. + private static string OptimismVerdict(double gap, double outOfFold) + { + if (!(outOfFold > 0)) return "The gap cannot be assessed."; + + double relative = gap / outOfFold; + return relative switch + { + < 0.05 => "The gap is negligible, so the fit describes the instrument rather than " + + "the particular peptides in this run.", + < 0.15 => "The gap is modest.", + < 0.30 => "The gap is substantial - the fit leans on the particular peptides in this " + + "run, so reusing this model elsewhere would do less well.", + _ => "The gap is large. The fit is thin, and this model should not be reused on " + + "other runs.", + }; + } + + private static void AppendSummaryTables( + StringBuilder html, TrainingStatistics? statistics, ErrorSummary? uncorrected, + MatchStatistics matchStatistics, IReadOnlyList inputFiles, + string toleranceDescription, ErrorScale scale) + { + html.Append("
"); + + html.Append("

Matching

"); + Row(html, "Spectra examined", matchStatistics.SpectraSeen.ToString("N0", CultureInfo.InvariantCulture)); + Row(html, "Fragments matched", matchStatistics.FragmentsMatched.ToString("N0", CultureInfo.InvariantCulture)); + Row(html, "Library precursors matched", matchStatistics.UniqueEntriesMatched.ToString("N0", CultureInfo.InvariantCulture)); + Row(html, "Tolerance", toleranceDescription); + html.Append("
"); + + if (statistics is not null) + { + html.Append("

Model

"); + Row(html, "Training rows", statistics.RowsTrain.ToString("N0", CultureInfo.InvariantCulture)); + Row(html, "Held out", statistics.RowsValidation.ToString("N0", CultureInfo.InvariantCulture)); + // Train and validation MAE come off the model, which is fitted in Th whatever the + // report is drawn in, so these two stay in Th and say so. + Row(html, "Train MAE", Format(statistics.TrainMae) + " Th"); + if (statistics.RowsValidation > 0) + Row(html, "Validation MAE", Format(statistics.ValidationMae) + " Th"); + html.Append("
"); + + ErrorSummary before = scale.Pick(statistics.Before, statistics.BeforePpm) ?? statistics.Before; + ErrorSummary after = scale.Pick(statistics.After, statistics.AfterPpm) ?? statistics.After; + + html.Append("

Mass error (").Append(scale.Unit).Append(")

"); + html.Append(""); + Row3(html, "Median absolute deviation", before.Mad, after.Mad, scale); + Row3(html, "Standard deviation", before.StdDev, after.StdDev, scale); + Row3(html, "Median", before.Median, after.Median, scale); + html.Append("
beforeafter
"); + } + else if (uncorrected is ErrorSummary summary) + { + html.Append("

Mass error

"); + Row(html, "Median absolute deviation", scale.Format(summary.Mad) + " " + scale.Unit); + Row(html, "Standard deviation", scale.Format(summary.StdDev) + " " + scale.Unit); + Row(html, "Median", scale.Format(summary.Median) + " " + scale.Unit); + Row(html, "Mean absolute error", scale.Format(summary.Mae) + " " + scale.Unit); + html.Append("
"); + } + + html.Append("

Input files

    "); + foreach (string file in inputFiles) + html.Append("
  • ").Append(Svg.Escape(Path.GetFileName(file))).Append("
    • "); + html.Append("
"); + + html.Append("
"); + } + + private static void Row(StringBuilder html, string label, string value) => + html.Append("").Append(Svg.Escape(label)).Append("") + .Append(Svg.Escape(value)).Append(""); + + private static void Row3( + StringBuilder html, string label, double before, double after, ErrorScale scale) => + html.Append("").Append(Svg.Escape(label)).Append("") + .Append(scale.Format(before)).Append("") + .Append(scale.Format(after)).Append(""); + + private static string Format(double value) => + double.IsNaN(value) ? "n/a" : value.ToString("0.0000", CultureInfo.InvariantCulture); + + /// + /// Readable name for a feature, for titles and axis labels. + /// + /// + /// The underscored names are the model's own vocabulary and stay that way everywhere they + /// are data - the model file, the CSV dumps, the parity comparison against the Python + /// implementation - because they are identifiers there and have to match exactly. This is + /// only what a reader sees. + /// + private static string[] PrettyNames(IReadOnlyList features) + { + var pretty = new string[features.Count]; + for (int i = 0; i < features.Count; i++) pretty[i] = Pretty(features[i]); + return pretty; + } + + private static string Pretty(string feature) => feature switch + { + "precursor_mz" => "precursor m/z", + "fragment_mz" => "fragment m/z", + "log_tic" => "log10 TIC", + "log_intensity" => "log10 peak intensity", + "absolute_time" => "acquisition time (s)", + "injection_time" => "injection time (s)", + "tic_injection_time" => "TIC x injection time", + "fragment_ions" => "fragment ions", + "rfa2_temp" => "RFA2 temperature", + "rfc2_temp" => "RFC2 temperature", + _ => PrettyWindow(feature), + }; + + /// + /// The space-charge features, whose names encode an m/z window: ions_above_1_2 counts the + /// ions between 1 and 2 Th above the fragment. + /// + private static string PrettyWindow(string feature) + { + string readable = feature.Replace('_', ' '); + (string Suffix, string Window)[] windows = + { + (" 0 1", " +0 to 1 Th"), (" 1 2", " +1 to 2 Th"), (" 2 3", " +2 to 3 Th"), + }; + + foreach ((string suffix, string window) in windows) + { + if (readable.EndsWith(suffix, StringComparison.Ordinal)) + return readable[..^suffix.Length] + window; + } + + return readable; + } + + private static void Figure(StringBuilder html, string? title, string? caption, string svg) + { + html.Append("
"); + if (title is not null) html.Append("

").Append(Svg.Escape(title)).Append("

"); + if (caption is not null) html.Append("
").Append(Svg.Escape(caption)).Append("
"); + html.Append(svg).Append("
"); + } + + // Light and dark are both handled, because a report that is emailed gets opened in + // whatever the recipient happens to use. + // White, deliberately, rather than following the reader's system theme. A QC report gets + // printed, pasted into a slide, and read next to figures from other tools, all of which + // assume a white page - and the density rasters cannot follow a theme anyway, so a dark + // surround would leave them sitting in a bright rectangle. + private const string Style = """ + :root { + --bg: #ffffff; --fg: #1a1d21; --muted: #5b6169; --grid: #e6e9ec; + --axis: #8b939b; --card: #ffffff; --border: #d9dee3; --accent: #3f7fbf; + } + * { box-sizing: border-box; } + body { + margin: 0; background: var(--bg); color: var(--fg); + font: 15px/1.55 system-ui, -apple-system, "Segoe UI", sans-serif; + } + main { max-width: 940px; margin: 0 auto; padding: 32px 20px 64px; } + h1 { font-size: 25px; margin: 0 0 4px; } + h2 { font-size: 16px; margin: 30px 0 10px; letter-spacing: .01em; } + h3 { font-size: 15px; margin: 0 0 4px; } + .sub { color: var(--muted); margin: 0 0 20px; font-size: 13px; } + .verdict { + background: #f6f8fa; border: 1px solid var(--border); border-left: 3px solid var(--accent); + border-radius: 4px; padding: 12px 14px; margin: 0 0 22px; + } + .note { color: var(--muted); font-size: 13px; margin: 0 0 16px; } + .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 18px; } + section { background: #f9fafb; border: 1px solid var(--border); border-radius: 4px; padding: 12px 14px; } + section h2 { margin-top: 0; } + table { width: 100%; border-collapse: collapse; font-size: 13px; } + td, th { padding: 3px 0; text-align: left; vertical-align: top; } + th { color: var(--muted); font-weight: 500; font-size: 12px; } + .num { text-align: right; font-variant-numeric: tabular-nums; } + .files { margin: 0; padding-left: 18px; font-size: 13px; word-break: break-all; } + figure { + margin: 20px 0 0; padding: 14px; background: #ffffff; + border: 1px solid var(--border); border-radius: 4px; + } + figcaption { color: var(--muted); font-size: 13px; margin: 8px 0 0; } + table.folds { font-size: 13px; } + table.folds th, table.folds td { padding: 4px 10px 4px 0; } + table.folds tr.total td { border-top: 1px solid var(--border); font-weight: 600; } + table.folds tr.spread td { color: var(--muted); } + svg { display: block; } + """; +} diff --git a/dotnet/MARS/Report/Svg.cs b/dotnet/MARS/Report/Svg.cs new file mode 100644 index 0000000..475bc74 --- /dev/null +++ b/dotnet/MARS/Report/Svg.cs @@ -0,0 +1,115 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// Minimal SVG construction for the QC report. + +using System; +using System.Globalization; +using System.Text; + +namespace MARS.Report; + +/// +/// A very small SVG writer: enough for axes, rectangles, polylines and text, and nothing +/// more. +/// +/// MARS draws its own charts rather than taking a plotting dependency. Every managed +/// charting library for .NET either wraps a native rasterizer or pulls in a large +/// dependency tree, and the port's stated goal is a binary with as little native code as +/// possible. SVG is text, so producing it costs nothing at runtime, it scales in an email +/// client, and it embeds directly in HTML with no base64 and no separate files. +/// +public sealed class Svg +{ + private readonly StringBuilder _body = new(); + + public Svg(int width, int height) + { + Width = width; + Height = height; + } + + public int Width { get; } + + public int Height { get; } + + private static string F(double value) => + // Two decimals is finer than a pixel at these sizes, and keeps the markup small: + // a density panel emits thousands of rectangles. + value.ToString("0.##", CultureInfo.InvariantCulture); + + public static string Escape(string text) => text + .Replace("&", "&", StringComparison.Ordinal) + .Replace("<", "<", StringComparison.Ordinal) + .Replace(">", ">", StringComparison.Ordinal) + .Replace("\"", """, StringComparison.Ordinal); + + public Svg Rect(double x, double y, double width, double height, string fill, string? extra = null) + { + if (width <= 0 || height <= 0) return this; + _body.Append(""); + return this; + } + + public Svg Line(double x1, double y1, double x2, double y2, string stroke, double width = 1, string? dash = null) + { + _body.Append(""); + return this; + } + + public Svg Polyline(ReadOnlySpan<(double X, double Y)> points, string stroke, double width = 1.5) + { + if (points.Length < 2) return this; + _body.Append(" 0) _body.Append(' '); + _body.Append(F(points[i].X)).Append(',').Append(F(points[i].Y)); + } + + _body.Append("\"/>"); + return this; + } + + public Svg Text( + double x, double y, string content, string anchor = "start", + double size = 11, string fill = "var(--fg)", bool bold = false, double rotate = 0) + { + _body.Append("').Append(Escape(content)).Append(""); + return this; + } + + /// + /// Places a raster image in the plot area. Used for the density layers, which are far + /// smaller as a compressed image than as one rectangle per cell. + /// + public Svg Image(double x, double y, double width, double height, string dataUri) + { + _body.Append(""); + return this; + } + + /// Serializes to an inline SVG element, sized to scale with its container. + public override string ToString() => + $"" + + _body + ""; +} diff --git a/dotnet/MARS/ResolutionMode.cs b/dotnet/MARS/ResolutionMode.cs new file mode 100644 index 0000000..f871b45 --- /dev/null +++ b/dotnet/MARS/ResolutionMode.cs @@ -0,0 +1,170 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using MARS.Core; + +namespace MARS.Cli; + +/// +/// Decides whether a run is high-resolution or unit-resolution, and what that implies for +/// the fragment tolerance and the units the QC report is written in. +/// +/// +/// +/// The vocabulary - unit, hram, auto - is Osprey's, so that someone +/// moving between the two tools does not have to learn a second word for the same idea. +/// Where MARS differs is what auto means: in Osprey it selects the configured +/// defaults, here it reads the instrument out of the mzML. The file already knows, and a +/// user who has to be told to pass a flag is a user who will eventually forget to. +/// +/// +/// Detection sets defaults only. An explicit --tolerance or --tolerance-ppm always wins, +/// because detection can be wrong on a file MARS has not seen the shape of before, and the +/// person at the terminal can be sure in a way a heuristic cannot. +/// +/// +public sealed class ResolutionMode +{ + public const double DefaultToleranceTh = 0.3; + public const double DefaultTolerancePpm = 10.0; + + private ResolutionMode() + { + } + + public MassAnalyzerClass Analyzer { get; private init; } + + /// True when the QC report should express mass error in ppm rather than Th. + public bool ReportInPpm => Analyzer == MassAnalyzerClass.HighResolution; + + /// + /// Warns when the matching window turns out to be far wider than the error in the data. + /// + /// + /// + /// A tolerance that is too wide fails silently: the window fills with peaks that are not + /// the fragment, the run completes, and the report looks ordinary. A tolerance that is too + /// narrow fails loudly, with too few matches to train on. So the dangerous direction is the + /// one worth checking for after the fact, and the data says which happened even when the + /// file's metadata does not. + /// + /// + /// This exists because instrument detection can come up empty. A ZenoTOF 8600 is the case + /// that prompted it: pwiz does not yet recognise the model, so it emits an instrument + /// configuration with no analyzer component and no filter string, MARS cannot tell what + /// recorded the run, and the fallback is a 0.3 Th window - about 760 ppm at m/z 400 on an + /// instrument whose real error is a few ppm. + /// + /// Median absolute deviation of the matched error, in Th. + /// + public static void WarnIfToleranceLooksTooWide( + MatchOptions options, double observedMad, double medianFragmentMz, Action log) + { + if (!(observedMad > 0) || !(medianFragmentMz > 0)) return; + + // The window in Th at a representative m/z, so ppm and Th tolerances compare. + double window = options.TolerancePpm > 0 + ? options.TolerancePpm * 1e-6 * medianFragmentMz + : options.MzToleranceTh; + if (!(window > 0)) return; + + // Fifty is loose on purpose. Trap data sits around 4 - a 0.08 Th spread inside a 0.3 Th + // window - so this cannot fire on the case MARS was built for. High-resolution data + // matched at a trap tolerance lands in the hundreds. + const double suspicious = 50.0; + double ratio = window / observedMad; + if (ratio < suspicious) return; + + log($" WARNING: the matching window is {ratio:N0}x the error actually in the data " + + $"({window:0.####} Th against a median absolute deviation of {observedMad:0.####} Th). " + + "That is the signature of a tolerance set for the wrong instrument: a window this " + + "wide admits peaks that are not the fragment, and the run will complete and report " + + "numbers regardless. If this is high-resolution data, re-run with --resolution hram " + + "or --tolerance-ppm."); + } + + /// + /// Reads --resolution, detects when it says auto, and fills in whichever tolerance the + /// user did not give. + /// + /// Mutated in place with the tolerance defaults for the mode. + /// + /// Registers the options this class reads, without acting on them. + /// + /// + /// learns that an option is real by watching it + /// be read, and it runs before any work so a typo costs a second. The resolution cannot be + /// decided that early - it needs the readers open to say what analyzer they found - so its + /// options are declared here instead. Without this, `--resolution` is rejected as a typo, + /// which is what happened. + /// + public static void Touch(CommandLineArgs args) + { + args.Has("resolution"); + args.Has("tolerance"); + args.Has("tolerance-ppm"); + } + + /// + /// What the input said its MS2 analyzer was. Each reader works this out for its own + /// format - an mzML from its instrumentConfiguration, a vendor file from the SDK - so this + /// takes the answer rather than reaching back into the file for it. Reading a .raw as if + /// it were mzML is how this silently fell back to a trap tolerance on Astral data. + /// + public static ResolutionMode Resolve( + CommandLineArgs args, MassAnalyzerClass detected, MatchOptions options, Action log) + { + bool toleranceGiven = args.Has("tolerance"); + bool ppmGiven = args.Has("tolerance-ppm"); + string requested = args.String("resolution")?.ToLowerInvariant() ?? "auto"; + + MassAnalyzerClass analyzer = requested switch + { + "unit" => MassAnalyzerClass.UnitResolution, + "hram" => MassAnalyzerClass.HighResolution, + "auto" => detected, + _ => throw new FormatException( + $"Option --resolution expects unit, hram or auto, got '{requested}'."), + }; + + if (toleranceGiven || ppmGiven) + { + // Say what was detected even when it changes nothing, so a mismatch between the + // instrument and the tolerance is visible in the log rather than only in the + // results. + if (analyzer != MassAnalyzerClass.Unknown) + log($" {MassAnalyzers.Describe(analyzer)} data; using the tolerance given on the command line"); + return new ResolutionMode { Analyzer = analyzer }; + } + + switch (analyzer) + { + case MassAnalyzerClass.HighResolution: + options.TolerancePpm = DefaultTolerancePpm; + options.MzToleranceTh = 0; + log($" high-resolution data; fragment tolerance {DefaultTolerancePpm:0.#} ppm " + + "(--tolerance or --tolerance-ppm to override)"); + break; + + case MassAnalyzerClass.UnitResolution: + options.MzToleranceTh = DefaultToleranceTh; + options.TolerancePpm = 0; + log($" unit-resolution data; fragment tolerance {DefaultToleranceTh:0.###} Th " + + "(--tolerance or --tolerance-ppm to override)"); + break; + + default: + options.MzToleranceTh = DefaultToleranceTh; + options.TolerancePpm = 0; + // A warning rather than a silent default: this is the case where MARS is + // guessing, and a 0.3 Th window on high-resolution data is wide enough to + // fill with wrong matches while still producing a report that looks fine. + log($" WARNING: could not tell the mass analyzer from the file; assuming " + + $"unit resolution and a {DefaultToleranceTh:0.###} Th tolerance. Pass " + + "--resolution hram or --tolerance-ppm if this is Orbitrap, TOF or Astral data."); + break; + } + + return new ResolutionMode { Analyzer = analyzer }; + } +} diff --git a/dotnet/MARS/ThreadCount.cs b/dotnet/MARS/ThreadCount.cs new file mode 100644 index 0000000..6e97dd1 --- /dev/null +++ b/dotnet/MARS/ThreadCount.cs @@ -0,0 +1,113 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; +using System.Globalization; + +namespace MARS.Cli; + +/// +/// Decides how many worker threads a run gets. +/// +/// +/// +/// One number drives all three parallel stages: the mzML writer, the pwiz spectrum list, and +/// the histogram build inside the boosting implementation. Matching is not among them - it +/// streams spectra in order on one thread - so a run's wall clock never falls in proportion +/// to this. +/// +/// +/// The default is every logical processor, which on a machine with simultaneous multithreading +/// is twice the physical core count. That is worth measuring rather than assuming, because the +/// usual advice is that the extra hardware threads do little for numeric work. On the reference +/// machine, an 8-core i9-9900K with 16 logical processors, correcting and rewriting one 1.2 GB +/// Stellar run: +/// +/// +/// threads 2 4 6 8 10 12 16 +/// seconds 150.5 77.4 52.5 45.4 42.8 38.7 36.8 +/// +/// +/// Scaling is near-perfect to 4 and keeps improving to the end: the 16 logical processors are +/// 24% faster than the 8 physical ones. So the default stays at every logical processor, and +/// capping at physical cores would cost most of a quarter of the throughput for nothing. +/// +/// +/// It is a shallow curve past 8, though - half the ideal speedup by 16 - and the writer drains +/// its results in order on one thread, which has to become the limit somewhere. Where that is +/// on a 64- or 128-core machine has not been measured, so no ceiling is imposed here: a guessed +/// one would be worse than none. The chosen number is reported instead, so anyone with such a +/// machine can see what they got and set --threads against it. +/// +/// +public static class ThreadCount +{ + /// What --threads accepts in place of a number. + public const string Automatic = "auto"; + + /// + /// Registers the option so the unknown-option check knows it is real. + /// + /// + /// Only needed where a command resolves the count after that check has run. Reading it + /// twice is harmless; not reading it before the check reports it as a typo. + /// + public static void Touch(CommandLineArgs args) => args.Has("threads"); + + /// + /// Resolves --threads to a concrete count, reporting what it settled on. + /// + /// Where the decision is reported. Null to decide silently. + /// Where an oversubscribed request is reported. + public static int Resolve(CommandLineArgs args, Action? log, Action? warn) + { + int available = Environment.ProcessorCount; + string? text = args.String("threads"); + + if (text is null || text.Equals(Automatic, StringComparison.OrdinalIgnoreCase)) + { + log?.Invoke( + $"Using {available} worker thread{(available == 1 ? string.Empty : "s")}, one per " + + "logical processor. --threads to change it."); + return available; + } + + // How the parser records an option given without a value, which is what + // `--threads -4` looks like to it: the -4 is read as the next option, not as a value. + if (text == "true") + { + throw new FormatException( + $"--threads was given no value. Pass a whole number or '{Automatic}'; note that " + + "a negative count has to be written --threads=-1 to keep it from being read " + + "as another option."); + } + + if (!int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out int requested)) + { + throw new FormatException( + $"--threads expects a whole number or '{Automatic}', got '{text}'."); + } + + if (requested < 1) + { + // Silently meaning "all of them" is the kind of default that hides a scripting + // mistake: --threads $N with N unset should say so, not quietly use the machine. + throw new FormatException( + $"--threads must be at least 1, got {requested}. Use '{Automatic}' for one per " + + "logical processor."); + } + + if (requested > available) + { + warn?.Invoke( + $"--threads {requested} is more than the {available} logical processors this " + + "machine has. The extra threads contend for the same cores rather than adding " + + "any, and the correction is unaffected either way."); + } + else + { + log?.Invoke($"Using {requested} of {available} worker threads, as asked."); + } + + return requested; + } +} diff --git a/dotnet/MARS/UnknownOptionException.cs b/dotnet/MARS/UnknownOptionException.cs new file mode 100644 index 0000000..f3d4539 --- /dev/null +++ b/dotnet/MARS/UnknownOptionException.cs @@ -0,0 +1,18 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. + +using System; + +namespace MARS.Cli; + +/// +/// A supplied option is not one the command understands. Raised before the command starts +/// work, and fatal - see for why this is not a +/// warning. +/// +public sealed class UnknownOptionException : Exception +{ + public UnknownOptionException(string message) + : base(message) + { + } +} diff --git a/dotnet/MARS/VerifyCommand.cs b/dotnet/MARS/VerifyCommand.cs new file mode 100644 index 0000000..b2645bf --- /dev/null +++ b/dotnet/MARS/VerifyCommand.cs @@ -0,0 +1,205 @@ +// Copyright (c) University of Washington 2026. Licensed under the MIT License. +// The passthrough acceptance gate: a null correction must round-trip a file with +// bit-identical decoded arrays, a valid index and a valid checksum. + +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using MARS.IO; + +namespace MARS.Cli; + +public static class VerifyCommand +{ + public static int Run(CommandLineArgs args) + { + if (args.Flag("help", "h")) + { + Console.Error.WriteLine(""" + Usage: mars verify [options] + + Round-trips a file through the passthrough writer applying a null correction + (decode and re-encode every m/z array without changing a value), then checks + that the result is equivalent to the input. + + This isolates the file-format work from the science. Run it before trusting + any calibrated output. + + Options: + -o, --output Where to write the round-tripped copy + (default: alongside the input, -verify.mzML) + --keep Keep the round-tripped file (default: delete it) + --threads Worker threads (default: auto, one per + logical processor) + --check-offsets N Index offsets to spot check (default: all) + -v, --verbose Verbose output + """); + return Program.ExitSuccess; + } + + Log.Verbose = args.Flag("verbose", "v"); + + string? inputPath = args.String("input", "i") ?? (args.Positional.Count > 0 ? args.Positional[0] : null); + if (inputPath is null) + { + Log.Error("No input file. Usage: mars verify "); + return Program.ExitInputError; + } + + if (!File.Exists(inputPath)) + { + Log.Error($"File not found: {inputPath}"); + return Program.ExitInputError; + } + + bool keep = args.Flag("keep"); + string outputPath = args.String("output", "o") + ?? Path.Combine( + Path.GetDirectoryName(Path.GetFullPath(inputPath)) ?? ".", + Path.GetFileNameWithoutExtension(inputPath) + "-verify.mzML"); + + // Refuse to write over the input. Verify deletes its output unless --keep, so + // pointing --output at the input would round-trip the file onto itself and then + // delete it - losing raw data to a command whose whole purpose is to prove nothing + // was lost. Compared on full paths so that a relative path and an absolute one to + // the same file are still caught. + // + // Case matters where the filesystem says it does. Linux distinguishes run.mzML from + // Run.mzML, and refusing that pair would be refusing a legitimate output path; + // Windows and macOS default to not distinguishing them, where treating them as + // different is how the input gets destroyed. The error the comparison can still make + // is to over-refuse on a case-insensitive Linux mount, which costs a rename rather + // than a file. + StringComparison pathComparison = OperatingSystem.IsLinux() + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + if (string.Equals(Path.GetFullPath(outputPath), Path.GetFullPath(inputPath), pathComparison)) + { + Log.Error( + "--output is the same file as the input. mars verify writes a round-tripped " + + "copy and deletes it unless --keep is given, so this would destroy the input. " + + "Choose a different --output."); + return Program.ExitInputError; + } + + int threads = ThreadCount.Resolve(args, Log.Info, Log.Warn); + int checkOffsets = args.Int("check-offsets") ?? 0; + + // Every option this command reads has been read by now, so a typo can be named rather + // than silently ignored. RejectUnknown only knows an option is real because something + // asked for it. + args.RejectUnknown(); + + var stopwatch = Stopwatch.StartNew(); + MzMLFileInfo info = MzMLFile.Inspect(inputPath); + Log.Info($"Input: {inputPath}"); + Log.Info($" {info.Length:N0} bytes, indexed={info.WasIndexed}, indexedmzML={info.IsIndexedMzML}"); + Log.Info($" acquisition start: {(info.AcquisitionStartTime is double t ? t.ToString("F3", CultureInfo.InvariantCulture) : "not recorded")}"); + + MzMLWriteResult write = MzMLWriter.Write( + info, outputPath, () => new NullMzTransform(), + new MzMLWriteOptions { MaxDegreeOfParallelism = threads }, + Log.Warn); + + Log.Info($"Wrote {outputPath}"); + Log.Info($" {write.OutputLength:N0} bytes, {write.SpectraSeen:N0} spectra, " + + $"{write.SpectraCorrected:N0} re-encoded, {write.ChromatogramsCopied:N0} chromatograms"); + Log.Info($" elapsed {stopwatch.Elapsed.TotalSeconds:F1} s"); + + var failures = 0; + + IndexValidationResult validation = MzMLValidator.Validate(outputPath, checkOffsets); + if (!validation.IsIndexed) + { + Log.Warn("Output has no index."); + } + else + { + Log.Info($"Index: {validation.SpectrumOffsets:N0} spectrum offsets, " + + $"{validation.ChromatogramOffsets:N0} chromatogram offsets"); + if (validation.BadOffsets.Count > 0) + { + failures++; + Log.Error($"{validation.BadOffsets.Count} index offsets do not land on their element:"); + foreach (string bad in validation.BadOffsets) Log.Error(" " + bad); + } + else + { + Log.Info("Index offsets: all land on the element they name"); + } + + if (validation.ChecksumPresent) + { + if (validation.ChecksumValid) + { + Log.Info($"SHA-1 fileChecksum: valid ({validation.RecordedChecksum})"); + } + else + { + failures++; + Log.Error($"SHA-1 fileChecksum invalid: recorded {validation.RecordedChecksum}, " + + $"computed {validation.ComputedChecksum}"); + } + } + } + + Log.Info("Comparing decoded arrays against the input..."); + MzMLComparison comparison = MzMLComparer.Compare(inputPath, outputPath); + Log.Info($" {comparison.SpectraCompared:N0} spectra, {comparison.MzValuesCompared:N0} peaks compared"); + + if (comparison.MzBitIdentical) + { + Log.Info(" m/z arrays: bit-identical"); + } + else + { + failures++; + Log.Error($" m/z arrays differ in {comparison.MzValuesDiffering:N0} values " + + $"(max |delta| {comparison.MaxAbsoluteMzDifference:R})"); + } + + if (comparison.IntensityBitIdentical) + { + Log.Info(" intensity arrays: bit-identical"); + } + else + { + failures++; + Log.Error($" intensity arrays differ in {comparison.IntensityValuesDiffering:N0} values"); + } + + if (comparison.SpectraOnlyInA != 0 || comparison.SpectraOnlyInB != 0) + { + failures++; + Log.Error($" spectrum count mismatch: {comparison.SpectraOnlyInA} only in input, " + + $"{comparison.SpectraOnlyInB} only in output"); + } + + foreach (string problem in comparison.Problems) Log.Error(" " + problem); + + if (!keep) + { + try + { + File.Delete(outputPath); + Log.Debug($"Deleted {outputPath}"); + } + catch (IOException ex) + { + Log.Warn($"Could not delete {outputPath}: {ex.Message}"); + } + } + + if (failures > 0) + { + Log.Error($"PASSTHROUGH VERIFICATION FAILED ({failures} checks)"); + return Program.ExitOutputValidationFailure; + } + + Console.Out.WriteLine("passthrough verification passed"); + Log.Info($"Total elapsed {stopwatch.Elapsed.TotalSeconds:F1} s"); + return Program.ExitSuccess; + } +} diff --git a/dotnet/Pwiz.props b/dotnet/Pwiz.props new file mode 100644 index 0000000..e1ea27e --- /dev/null +++ b/dotnet/Pwiz.props @@ -0,0 +1,48 @@ + + + + + + $(MSBuildThisFileDirectory)..\..\..\ProteoWizard\pwiz\pwiz-sharp + $(PwizSharpDir)\pwiz\src\MsData\MsData.csproj + $(PwizSharpDir)\pwiz\src\Vendor\Thermo\Thermo.csproj + $(PwizSharpDir)\pwiz\src\Vendor\Bruker\Bruker.csproj + $(PwizSharpDir)\pwiz\src\Vendor\Sciex\Sciex.csproj + $(PwizSharpDir)\pwiz\src\Vendor\Sciex\Wiff2\Sciex.Wiff2.csproj + $(PwizSharpDir)\pwiz\src\Vendor\Sciex\OfxLoggingStub\OfxLoggingStub.csproj + + + true + true + false + + + + + false + + + diff --git a/dotnet/README.md b/dotnet/README.md new file mode 100644 index 0000000..75dcf24 --- /dev/null +++ b/dotnet/README.md @@ -0,0 +1,116 @@ +# MARS for .NET + +C# port of MARS (Mass Accuracy Recalibration System). This file covers the source tree; for +using the tool, start at the [top-level README](../README.md). + +- [Algorithm](../docs/algorithm.md) - what MARS computes and why +- [mzML passthrough](../docs/mzml-passthrough.md) - how output files are written +- [Port specification](../docs/dotnet-port-spec.md) - design, acceptance gates, and what the + port found in the Python implementation + +## Building + +``` +cd dotnet +dotnet build -c Release +dotnet test +``` + +Targets `net8.0` by default, which runs unchanged on the .NET 9 and .NET 10 runtimes. +With a .NET 10 SDK installed, build the full matrix: + +``` +dotnet build -c Release -p:MarsIncludeNet10=true +``` + +Not `-p:MarsTargetFrameworks="net8.0;net10.0"`: a semicolon-separated list cannot survive the +command line. The shell eats the quotes, and escaping the separator as `%3B` makes MSBuild read +the whole string as one target framework name. + +One NuGet reference: `Parquet.Net`, for DIA-NN libraries. It carries a native +compression library (`nironcompress`), which is the only native code in the tree. +`MARS.Core` and `MARS.OspreyML` have no package references and are pure managed; the +dependency is confined to `MARS.IO`. + +BiblioSpec `.blib` files are read through a managed SQLite reader written for this purpose +rather than `Microsoft.Data.Sqlite`, so that path adds no native code of its own. + +## Commands + +``` +mars calibrate Learn a calibration from library matches and write corrected mzML +mars apply Apply an existing model to more files +mars qc Report mass accuracy without training or writing +mars verify Round-trip a file with a null correction and check it +mars compare Compare two mzML files on decoded values +``` + +Every command takes `--help`. Diagnostics go to stderr so stdout stays pipeable. +Exit codes: 0 success, 1 input error, 2 insufficient training data, 3 output validation +failure. + +### Typical run + +``` +mars calibrate \ + --mzml-dir raw/ \ + --prism-csv skyline-report.csv \ + --temperature-dir temperature_csvs/ \ + --output-dir corrected/ +``` + +Writes `{input}-mars.mzML` per file, plus `mars_model.json` and `mars_qc_summary.txt`. + +### Before trusting any output + +``` +mars verify raw/run.mzML +``` + +Round-trips the file through the writer applying a null correction, then checks that the +result decodes to bit-identical m/z and intensity arrays, that every index offset lands +on the element it names, and that the SHA-1 checksum validates. This isolates the +file-format work from the science; run it first when something looks wrong. + +## Layout + +| Project | Contents | +|---|---| +| `MARS.Core` | Domain types, fragment matching, feature extraction, the calibration model, correction | +| `MARS.IO` | mzML passthrough reader/writer, library readers, managed SQLite | +| `MARS.OspreyML` | Compiles the vendored Osprey.ML sources | +| `MARS` | CLI | +| `MARS.Test` | Unit and contract tests | + +## Vendored Osprey.ML + +`third_party/Osprey.ML/` holds a copy of the gradient boosted trees from +`pwiz_tools/Osprey/Osprey.ML`. **Osprey.ML owns that code.** MARS carries a copy only +because pwiz has no package feed to consume yet. + +Do not edit the vendored files. Fix things upstream in pwiz, then: + +``` +pwsh -File ./scripts/sync-osprey-ml.ps1 -PwizPath D:\Dev\pwiz # report drift +pwsh -File ./scripts/sync-osprey-ml.ps1 -PwizPath D:\Dev\pwiz -Apply # pull it down +``` + +`UPSTREAM.json` records the source commit and a SHA-256 per file, and `MARS.Test` fails +when they stop matching, so an accidental local edit becomes a visible test failure +rather than a silent fork. + +## Determinism + +MARS writes m/z values into files that get reprocessed and compared, so identical input +must produce a bit-identical output, at any thread count, on any platform. The +guarantees: + +- Histogram accumulation parallelizes across FEATURES only, so each histogram is summed + in ascending row order by one thread. +- Subsampling draws from `XorShift64`, seeded, never `System.Random`. +- Split selection walks features and bins in ascending order and takes a new best only + on a strict improvement, so ties resolve to the lowest (feature, bin). +- Row partitioning is stable. +- Inference carries no cross-row accumulation, so parallelizing it cannot change a value. + +CI asserts this by writing the same file at 1 and 16 threads and comparing bytes. diff --git a/dotnet/pwiz-sharp.json b/dotnet/pwiz-sharp.json new file mode 100644 index 0000000..0df46d9 --- /dev/null +++ b/dotnet/pwiz-sharp.json @@ -0,0 +1,46 @@ +{ + "$comment": [ + "The pwiz-sharp commit MARS builds its vendor support against.", + "", + "pwiz-sharp is ProteoWizard's .NET 8 port of the C++ core. MARS uses it to read Thermo,", + "Bruker and Sciex data and to write mzXML, mzMLb and mgf. It is NOT vendored - it is far", + "too large, and its vendor SDKs are license-gated - so this records which commit to check", + "out rather than a copy of the code.", + "", + "Pinned rather than tracking the branch because PR #4178 is an unmerged draft: it rebases", + "and force-pushes, and an unpinned build would change underneath a release without anyone", + "choosing to. CI and the release workflow both read this file.", + "", + "To move to a newer commit: update commit and commitSubject here, run the build and the", + "tests against it locally, and say in the release notes what changed. There is no drift", + "guard the way third_party/Osprey.ML has one, because nothing is copied to drift from.", + "", + "This goes away when pwiz-sharp merges to master. At that point MARS should stop", + "carrying its own copies of the vendor SDKs and use the ones an installed", + "Skyline-Daily, Skyline or msconvert already provides, in that order - daily first", + "because ClickOnce updates it fastest, msconvert last because it never updates", + "itself. Whichever is found, check the SDK version before trusting it. See", + "docs/open-questions.md for the discovery recipe and for why that cannot be done", + "today: Skyline currently ships a 5.0.0.93 .NET Framework build where pwiz-sharp", + "needs 8.0.6.0." + ], + "repository": "https://github.com/ProteoWizard/pwiz", + "branch": "chambem2/pwiz-sharp", + "commit": "1d26ed967ba701d3c090ee882c4e526aa72c4126", + "commitSubject": "pwiz: Added opt-in parallel decoding to the pwiz-sharp mzML reader (#4590)", + "pullRequest": "https://github.com/ProteoWizard/pwiz/pull/4178", + "pinnedOn": "2026-08-21", + + "$notes": [ + "Needs the FULL working tree, not a sparse checkout of pwiz-sharp/. The Bruker reader", + "takes its archives from pwiz_aux/msrc/utility and pulls VC90 CRT files from", + "pwiz_tools/Shared/Lib, and Common.csproj embeds three .obo files from pwiz/data/common.", + "", + "Needs a global.json pinning SDK 8, which the branch does not carry. Without one, a", + "machine with a newer SDK fails to resolve one for the nested dotnet run that generates", + "the vendor pins. CI writes it; see .github/workflows/dotnet.yml.", + "", + "Vendor SDKs are gated behind -p:IAgreeToVendorLicenses=true. The archive password is in", + "the repository, so this is a click-through acknowledgement rather than a secret." + ] +} diff --git a/dotnet/scripts/compare_matches.py b/dotnet/scripts/compare_matches.py new file mode 100644 index 0000000..05d9364 --- /dev/null +++ b/dotnet/scripts/compare_matches.py @@ -0,0 +1,207 @@ +"""Difference two match dumps row by row and column by column. + +Takes the CSV written by ``mars calibrate --dump-matches`` and the one written by +dump_python_matches.py, joins them on the rows they both found, and reports where the two +implementations disagree. + + python compare_matches.py --csharp cs.csv --python py.csv + +The join key is (scan_number, ion_annotation, expected_mz). Peptide sequence is +deliberately not part of it: the two implementations carry the modified sequence in +whatever form their library reader produced, and a formatting difference there is not a +calibration difference. Rows whose key appears more than once on either side are reported +and excluded, since there is no way to say which copy pairs with which. + +Exit status is 1 if any column disagrees beyond its tolerance, so this can gate a build. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +BASE_KEY = ["scan_number", "ion_annotation", "expected_mz_key"] + +# A handful of precursors appear in more than one block of a PRISM report, so the same +# fragment can be matched twice in one scan through two library entries. Both +# implementations produce those duplicates, so rather than discarding the rows, order +# each group identically on both sides and pair them off by position. If the two ever +# produced different multisets the surplus would show up as an unmatched row. +TIEBREAK = ["observed_mz", "delta_mz", "observed_intensity"] +KEY = BASE_KEY + ["occurrence"] + +# Per-column absolute tolerance. Most of these are computed by the same arithmetic on both +# sides and should agree to the last bit; the tolerance exists to absorb the decimal +# round-trip through CSV, not to excuse a real difference. +EXACT = 0.0 +DEFAULT_TOLERANCE = 1e-9 + +TOLERANCES = { + # Intensities are float32 in the mzML and stay float32 through the C# reader, so a + # value that passes through float64 arithmetic on one side only can differ in the last + # float32 digit. + "observed_intensity": 1e-3, + "log_intensity": 1e-9, + "log_tic": 1e-9, + # Sums over thousands of float32 intensities, where accumulation order is visible. + "fragment_ions": 1e-3, + "ions_above_0_1": 1e-3, + "ions_above_1_2": 1e-3, + "ions_above_2_3": 1e-3, + "ions_below_0_1": 1e-3, + "ions_below_1_2": 1e-3, + "ions_below_2_3": 1e-3, + "tic_injection_time": 1e-3, +} + + +def load(path: Path, label: str) -> pd.DataFrame: + # float_precision="round_trip" is not optional here. The default parser is faster and + # drops the last digit, which invents differences of ~1e-16 in every column and buries + # the real ones. + frame = pd.read_csv(path, float_precision="round_trip") + if "expected_mz" not in frame.columns: + raise SystemExit(f"{label} dump has no expected_mz column: {path}") + # A float is a poor join key. Round to a tenth of a milli-Thomson, far finer than any + # real difference between two theoretical m/z values and far coarser than float noise. + frame["expected_mz_key"] = frame["expected_mz"].round(4) + return frame + + +def number_occurrences(frame: pd.DataFrame, label: str) -> pd.DataFrame: + """Make a repeated key unique by position within its group.""" + duplicated = int(frame.duplicated(subset=BASE_KEY, keep=False).sum()) + if duplicated: + print(f" {label}: {duplicated:,} rows share a key; paired by position within the group") + frame = frame.sort_values(BASE_KEY + TIEBREAK, kind="mergesort") + frame["occurrence"] = frame.groupby(BASE_KEY, sort=False).cumcount() + return frame + + +def describe(value: float) -> str: + """Shortest round-trip form. repr() on a numpy scalar prints np.float64(...).""" + return "NaN" if pd.isna(value) else repr(float(value)) + + +def compare_column(merged: pd.DataFrame, column: str) -> dict: + left = pd.to_numeric(merged[f"{column}_cs"], errors="coerce").to_numpy(dtype=float) + right = pd.to_numeric(merged[f"{column}_py"], errors="coerce").to_numpy(dtype=float) + + left_nan, right_nan = np.isnan(left), np.isnan(right) + # Both undefined counts as agreement: NaN is how an undefined ratio is represented and + # the row is dropped on it downstream, so the two agree about the row's fate. + nan_mismatch = int((left_nan != right_nan).sum()) + + both = ~left_nan & ~right_nan + if not both.any(): + return {"n": 0, "max_abs": 0.0, "over": 0, "nan_mismatch": nan_mismatch} + + difference = np.abs(left[both] - right[both]) + tolerance = TOLERANCES.get(column, DEFAULT_TOLERANCE) + return { + "n": int(both.sum()), + "max_abs": float(difference.max()), + "over": int((difference > tolerance).sum()), + "nan_mismatch": nan_mismatch, + "tolerance": tolerance, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--csharp", required=True, type=Path) + parser.add_argument("--python", required=True, type=Path) + parser.add_argument( + "--examples", type=int, default=3, + help="worst-offending rows to print per failing column") + args = parser.parse_args() + + csharp = load(args.csharp, "C#") + python = load(args.python, "Python") + + print("Row counts") + print(f" C# {len(csharp):,}") + print(f" Python {len(python):,}") + print() + + print("Repeated keys") + csharp = number_occurrences(csharp, "C#") + python = number_occurrences(python, "Python") + print() + + merged = csharp.merge(python, on=KEY, how="inner", suffixes=("_cs", "_py")) + only_csharp = len(csharp) - len(merged) + only_python = len(python) - len(merged) + + print("Row agreement") + print(f" matched by both {len(merged):,}") + print(f" C# only {only_csharp:,}") + print(f" Python only {only_python:,}") + print() + + if merged.empty: + print("No rows in common; nothing to compare.") + return 1 + + shared = sorted( + {c[:-3] for c in merged.columns if c.endswith("_cs")} + & {c[:-3] for c in merged.columns if c.endswith("_py")} + ) + # entry_index and fragment_index are internal to the C# library layout and have no + # Python counterpart, so they never appear on both sides. peptide is text. + shared = [c for c in shared if c not in ("peptide",)] + + print(f"{'column':<26} {'n':>9} {'max abs diff':>16} {'over tol':>9} {'NaN mismatch':>13}") + print("-" * 78) + + failed = [] + for column in shared: + result = compare_column(merged, column) + flag = "" + if result["over"] or result["nan_mismatch"]: + flag = " <-- differs" + failed.append(column) + print( + f"{column:<26} {result['n']:>9,} {result['max_abs']:>16.3e} " + f"{result['over']:>9,} {result['nan_mismatch']:>13,}{flag}") + + if only_csharp or only_python: + failed.append("row set") + + print() + if not failed: + print("Every shared column agrees within tolerance, on every row both found.") + return 0 + + print(f"Disagreements: {', '.join(failed)}") + for column in failed: + if column == "row set" or args.examples <= 0: + continue + left = pd.to_numeric(merged[f"{column}_cs"], errors="coerce") + right = pd.to_numeric(merged[f"{column}_py"], errors="coerce") + + # A row where only one side is NaN has no magnitude to rank by, and it is the more + # interesting failure - the two disagree about whether the feature is defined at + # all, which decides whether the row survives to training - so show those first. + undefined_on_one_side = left.isna() != right.isna() + rows = list(merged.index[undefined_on_one_side][: args.examples]) + magnitude = (left - right).abs() + magnitude = magnitude[magnitude > 0] + rows += [i for i in magnitude.nlargest(args.examples).index if i not in rows] + + print(f"\n {column}:") + for index in rows: + print( + f" scan {merged.at[index, 'scan_number']} " + f"{merged.at[index, 'ion_annotation']} " + f"expected {merged.at[index, 'expected_mz_key']}: " + f"C# {describe(left[index])} vs Python {describe(right[index])}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dotnet/scripts/compare_models.py b/dotnet/scripts/compare_models.py new file mode 100644 index 0000000..c872b14 --- /dev/null +++ b/dotnet/scripts/compare_models.py @@ -0,0 +1,174 @@ +"""Compare the C# gradient boosted trees against Python's XGBoost on identical data. + +The two implementations will never agree tree for tree - they are different codebases with +different split-finding and different tie-breaking. The question that matters is whether +they learn the same function, and whether either corrects the data better. + +This trains XGBoost on exactly the rows the C# model was trained on, taken from the C# +prediction dump, and compares the two predictions row by row. + + mars calibrate --mzml run.mzML --prism-csv report.csv --no-dedupe-library \\ + --validation-split 0 --no-recalibrate --dump-predictions cs.csv --output-dir out/ + + python compare_models.py --csharp cs.csv + +Use --validation-split 0. Otherwise the C# model is fitted on a subset chosen by its own +splitter and scored on rows it never saw, while XGBoost here would see everything, and the +difference between the two would be mostly the split rather than the implementation. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import xgboost as xgb + +# Columns the dump carries that are not model features. +NON_FEATURES = { + "scan_number", "retention_time", "entry_index", "fragment_index", "peptide", + "peptide_group", "ion_annotation", "expected_mz", "observed_mz", "delta_mz", + "observed_intensity", "predicted_delta_mz", "residual", +} + +# MzCalibrator's defaults, which are XGBoost's defaults apart from these four. +N_ESTIMATORS = 100 +MAX_DEPTH = 6 +LEARNING_RATE = 0.1 +SEED = 42 + + +def robust(values: np.ndarray) -> tuple[float, float]: + """Standard deviation and median absolute deviation, in Th.""" + median = float(np.median(values)) + return float(np.std(values)), float(np.median(np.abs(values - median))) + + +def assign_folds(groups: np.ndarray, folds: int) -> np.ndarray: + """Reproduce MARS's fold assignment exactly. + + Distinct peptide groups sorted ascending, then dealt round-robin. No PRNG, so both + implementations land on the same split from the same input, and the two models are + compared on identical training and held-out rows rather than merely similar ones. + """ + distinct = np.unique(groups) + fold_of_group = {g: i % folds for i, g in enumerate(distinct)} + return np.array([fold_of_group[g] for g in groups], dtype=int) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--csharp", required=True, type=Path, help="--dump-predictions output") + parser.add_argument("--n-estimators", type=int, default=N_ESTIMATORS) + parser.add_argument("--max-depth", type=int, default=MAX_DEPTH) + parser.add_argument("--learning-rate", type=float, default=LEARNING_RATE) + parser.add_argument("--seed", type=int, default=SEED) + parser.add_argument("--cv-folds", type=int, default=5, + help="peptide-grouped folds for the Python side; 0 to skip") + args = parser.parse_args() + + frame = pd.read_csv(args.csharp, float_precision="round_trip") + if "predicted_delta_mz" not in frame.columns: + raise SystemExit( + f"{args.csharp} has no predicted_delta_mz column. It must come from " + "--dump-predictions, not --dump-matches.") + + features = [c for c in frame.columns if c not in NON_FEATURES] + + # The C# model scores NaN for a row with any undefined feature, which is exactly the set + # of rows it could not train on. Use the same set so both see identical data. + usable = frame["predicted_delta_mz"].notna() & frame[features].notna().all(axis=1) + used = frame[usable] + + print(f"Rows in dump {len(frame):,}") + print(f"Rows both can use {len(used):,}") + print(f"Features {len(features)} ({', '.join(features)})") + print() + + x = used[features].to_numpy(dtype=float) + y = used["delta_mz"].to_numpy(dtype=float) + + # Both implementations weight by observed intensity normalized to mean 1. The + # normalization is not cosmetic: reg_lambda and min_child_weight are thresholds on + # summed hessians, which under squared error are summed weights. + weight = used["observed_intensity"].to_numpy(dtype=float) + weight = weight / weight.mean() + + def fit(rows: np.ndarray) -> xgb.XGBRegressor: + model = xgb.XGBRegressor( + n_estimators=args.n_estimators, + max_depth=args.max_depth, + learning_rate=args.learning_rate, + random_state=args.seed, + n_jobs=-1, + objective="reg:squarederror", + ) + model.fit(x[rows], y[rows], sample_weight=weight[rows], verbose=False) + return model + + # ---- peptide-grouped cross-validation, the same split MARS used ------------------- + if args.cv_folds >= 2 and "peptide_group" in used.columns: + groups = used["peptide_group"].to_numpy() + fold_of_row = assign_folds(groups, args.cv_folds) + out_of_fold = np.empty_like(y) + + print(f"Python cross-validation ({args.cv_folds} folds over " + f"{len(np.unique(groups)):,} peptides, same split as MARS)") + for fold in range(args.cv_folds): + held_out = fold_of_row == fold + out_of_fold[held_out] = fit(~held_out).predict(x[held_out]).astype(float) + residual = y[held_out] - out_of_fold[held_out] + print(f" fold {fold + 1}: {held_out.sum():>8,} rows, " + f"MAD {np.median(np.abs(residual - np.median(residual))):.4f} Th") + + oof_std, oof_mad = robust(y - out_of_fold) + print(f" pooled out-of-fold: MAD {oof_mad:.4f} Th, std {oof_std:.4f} Th") + print(" Compare against the out-of-fold MAD in mars_qc_summary.txt: both models were") + print(" trained on the same rows and scored on the same held-out peptides.") + print() + + model = fit(np.arange(len(y))) + python_prediction = model.predict(x).astype(float) + csharp_prediction = used["predicted_delta_mz"].to_numpy(dtype=float) + + difference = csharp_prediction - python_prediction + correlation = float(np.corrcoef(csharp_prediction, python_prediction)[0, 1]) + + print("Predictions on the same rows (both trained on everything, so in-sample)") + print(f" Pearson r {correlation:.6f}") + print(f" mean difference {difference.mean():+.6f} Th") + print(f" RMS difference {np.sqrt((difference ** 2).mean()):.6f} Th") + print(f" median absolute difference {np.median(np.abs(difference)):.6f} Th") + print(f" 95th percentile |difference| {np.percentile(np.abs(difference), 95):.6f} Th") + print(f" max |difference| {np.abs(difference).max():.6f} Th") + print() + + # The number that decides whether a difference matters: what is left after correcting. + before_std, before_mad = robust(y) + cs_std, cs_mad = robust(y - csharp_prediction) + py_std, py_mad = robust(y - python_prediction) + + print("Residual after correction (the number that matters)") + print(f" {'':<12} {'std (Th)':>10} {'MAD (Th)':>10}") + print(f" {'uncorrected':<12} {before_std:>10.4f} {before_mad:>10.4f}") + print(f" {'C#':<12} {cs_std:>10.4f} {cs_mad:>10.4f}") + print(f" {'Python':<12} {py_std:>10.4f} {py_mad:>10.4f}") + print() + print(f" spread reduction C# {100 * (1 - cs_std / before_std):5.1f}% " + f"Python {100 * (1 - py_std / before_std):5.1f}%") + print(f" MAD reduction C# {100 * (1 - cs_mad / before_mad):5.1f}% " + f"Python {100 * (1 - py_mad / before_mad):5.1f}%") + print() + + # A prediction difference only matters relative to the error being corrected. Stating it + # as a fraction of the uncorrected spread is what says whether it is worth caring about. + relative = np.sqrt((difference ** 2).mean()) / before_std + print(f"RMS prediction difference is {100 * relative:.1f}% of the uncorrected spread.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dotnet/scripts/dump_python_matches.py b/dotnet/scripts/dump_python_matches.py new file mode 100644 index 0000000..30886df --- /dev/null +++ b/dotnet/scripts/dump_python_matches.py @@ -0,0 +1,162 @@ +"""Dump the Python implementation's match table in the same CSV schema as ``mars +calibrate --dump-matches``, so the two can be differenced row by row. + +The C# implementation agrees with the Python one on aggregate numbers - match counts and +the spread of the corrected error. That says nothing about whether any individual feature +is computed the same way, and the model leans hardest on features that aggregate +statistics cannot see. This script produces the other half of that comparison. + +Usage: + + python dump_python_matches.py --mzml run.mzML --prism-csv report.csv --out py.csv + +Then, having produced the C# side with + + mars calibrate --mzml run.mzML --prism-csv report.csv --no-dedupe-library \\ + --no-recalibrate --dump-matches cs.csv + +compare them with compare_matches.py. + +Pass --no-dedupe-library on the C# side: the Python implementation does not collapse +transitions that repeat across replicates, so leaving C# to dedupe would compare +different row sets. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +import pandas as pd + +from mars.library import load_prism_library +from mars.matching import match_library_to_spectra +from mars.mzml import read_dia_spectra + +# Same order as MatchDumpWriter.KeyColumns, so a header diff is a real disagreement rather +# than a column-order artifact. +KEY_COLUMNS = [ + "scan_number", + "retention_time", + "peptide", + "ion_annotation", + "expected_mz", + "observed_mz", + "delta_mz", + "observed_intensity", +] + +# Features carried on the match rows themselves. The adjacent_ratio_* features are derived +# later, in MzCalibrator._prepare_features, and are added below. +MATCH_FEATURES = [ + "precursor_mz", + "fragment_mz", + "log_tic", + "log_intensity", + "absolute_time", + "injection_time", + "tic_injection_time", + "fragment_ions", + "ions_above_0_1", + "ions_above_1_2", + "ions_above_2_3", + "ions_below_0_1", + "ions_below_1_2", + "ions_below_2_3", +] + +RATIO_SOURCES = [ + ("adjacent_ratio_0_1", "ions_above_0_1"), + ("adjacent_ratio_1_2", "ions_above_1_2"), + ("adjacent_ratio_2_3", "ions_above_2_3"), + ("adjacent_ratio_below_0_1", "ions_below_0_1"), + ("adjacent_ratio_below_1_2", "ions_below_1_2"), + ("adjacent_ratio_below_2_3", "ions_below_2_3"), +] + + +def add_derived_features(matches: pd.DataFrame) -> pd.DataFrame: + """Reproduce the ratio features that MzCalibrator derives at fit time. + + They are computed there rather than during matching, so a dump of the raw match table + would be missing exactly the two features the model weights most heavily. The + definition here must track ``MzCalibrator._prepare_features``; the guard on + ``fragment_ions > 0`` is what makes the ratio undefined, and the row is dropped + downstream on the resulting NaN. + """ + if "fragment_ions" not in matches.columns: + return matches + + defined = matches["fragment_ions"] > 0 + for ratio, source in RATIO_SOURCES: + if source in matches.columns: + matches[ratio] = (matches[source] / matches["fragment_ions"]).where(defined) + return matches + + +def rebase_absolute_time(matches: pd.DataFrame) -> pd.DataFrame: + """Subtract the earliest acquisition, which is what the model is trained on. + + The C# implementation re-bases after reading every input file and dumps the re-based + column, so the comparison has to be against the same quantity. + """ + if "absolute_time" in matches.columns and matches["absolute_time"].notna().any(): + matches["absolute_time"] = matches["absolute_time"] - matches["absolute_time"].min() + return matches + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mzml", required=True, type=Path, help="one mzML file") + parser.add_argument("--prism-csv", required=True, type=Path, help="Skyline PRISM report") + parser.add_argument("--out", required=True, type=Path, help="output CSV") + parser.add_argument("--tolerance", type=float, default=0.3, help="m/z tolerance in Th") + parser.add_argument("--min-intensity", type=float, default=500.0) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s") + log = logging.getLogger("dump") + + log.info("Loading PRISM library: %s", args.prism_csv) + library = load_prism_library(args.prism_csv, mzml_filename=args.mzml.name) + log.info("%d library entries", len(library)) + + log.info("Matching: %s", args.mzml.name) + matches = match_library_to_spectra( + library, + read_dia_spectra(args.mzml), + mz_tolerance=args.tolerance, + min_intensity=args.min_intensity, + show_progress=False, + ) + log.info("%d matches", len(matches)) + + if matches.empty: + log.error("No matches; nothing to compare.") + return 1 + + matches = rebase_absolute_time(matches) + matches = add_derived_features(matches) + + matches = matches.rename(columns={"peptide_sequence": "peptide"}) + columns = [c for c in KEY_COLUMNS if c in matches.columns] + columns += [c for c in MATCH_FEATURES if c in matches.columns] + columns += [r for r, _ in RATIO_SOURCES if r in matches.columns] + + missing = [c for c in KEY_COLUMNS if c not in matches.columns] + if missing: + log.warning("Match table has no %s; those columns will be absent", ", ".join(missing)) + + args.out.parent.mkdir(parents=True, exist_ok=True) + # No float_format: pandas writes the shortest representation that round-trips, which + # is what a comparison at m/z precision needs. Do not be tempted by "%r" - under + # numpy 2 that writes "np.float64(658.3)" and the file stops being numeric. + matches[columns].to_csv(args.out, index=False) + log.info("Wrote %d rows, %d columns to %s", len(matches), len(columns), args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dotnet/scripts/sync-osprey-ml.ps1 b/dotnet/scripts/sync-osprey-ml.ps1 new file mode 100644 index 0000000..4de26f7 --- /dev/null +++ b/dotnet/scripts/sync-osprey-ml.ps1 @@ -0,0 +1,158 @@ +#Requires -Version 7 +<# +.SYNOPSIS + Re-syncs the vendored Osprey.ML sources from a ProteoWizard/pwiz checkout. + +.DESCRIPTION + MARS vendors Osprey.ML's gradient boosted trees rather than referencing an assembly, + because pwiz has no package feed to consume yet. Osprey.ML remains the owner: bugs get + fixed upstream, then pulled down with this script. + + Without -Apply the script only reports. With -Apply it copies the upstream file, + re-extracts the XorShift64 fragment, and rewrites the hashes in UPSTREAM.json. + +.PARAMETER PwizPath + Path to a pwiz checkout, e.g. D:\Dev\pwiz. + +.PARAMETER Apply + Write the changes instead of only reporting them. + +.EXAMPLE + pwsh -File ./scripts/sync-osprey-ml.ps1 -PwizPath D:\Dev\pwiz + pwsh -File ./scripts/sync-osprey-ml.ps1 -PwizPath D:\Dev\pwiz -Apply +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$PwizPath, + + [switch]$Apply +) + +$ErrorActionPreference = 'Stop' + +$vendorDir = Join-Path $PSScriptRoot '..\third_party\Osprey.ML' | Resolve-Path +$manifestPath = Join-Path $vendorDir 'UPSTREAM.json' +$manifest = Get-Content $manifestPath -Raw | ConvertFrom-Json + +$ospreyMl = Join-Path $PwizPath 'pwiz_tools\Osprey\Osprey.ML' +if (-not (Test-Path $ospreyMl)) { + throw "Not a pwiz checkout: $ospreyMl does not exist" +} + +function Get-TextHash([string]$text) { + $bytes = [System.Text.Encoding]::UTF8.GetBytes($text) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + return [System.BitConverter]::ToString($sha.ComputeHash($bytes)).Replace('-', '') + } + finally { + $sha.Dispose() + } +} + +# Pull the XorShift64 class out of LinearSvmClassifier.cs by brace matching, so the +# fragment is extracted the same way every time rather than by hand. +function Get-XorShiftFragment([string]$sourcePath) { + $lines = [System.IO.File]::ReadAllLines($sourcePath) + $start = -1 + for ($i = 0; $i -lt $lines.Length; $i++) { + if ($lines[$i] -match '^\s*public class XorShift64\b') { + $start = $i + break + } + } + if ($start -lt 0) { + throw "XorShift64 not found in $sourcePath. The fragment guard needs updating." + } + + # Walk back over the preceding doc comment block. + $docStart = $start + while ($docStart -gt 0 -and $lines[$docStart - 1] -match '^\s*///') { + $docStart-- + } + + $depth = 0 + $end = -1 + for ($i = $start; $i -lt $lines.Length; $i++) { + $depth += ([regex]::Matches($lines[$i], '\{')).Count + $depth -= ([regex]::Matches($lines[$i], '\}')).Count + if ($depth -eq 0 -and $i -gt $start) { + $end = $i + break + } + } + if ($end -lt 0) { + throw "Unbalanced braces walking XorShift64 in $sourcePath" + } + + return ($lines[$docStart..$end] -join "`r`n") +} + +$changed = $false + +foreach ($file in $manifest.files) { + $vendoredPath = Join-Path $vendorDir $file.vendored + $upstreamPath = Join-Path $PwizPath ($file.upstream -replace '/', '\') + + if (-not (Test-Path $upstreamPath)) { + Write-Host "MISSING upstream: $($file.upstream)" -ForegroundColor Red + $changed = $true + continue + } + + $currentHash = (Get-FileHash $vendoredPath -Algorithm SHA256).Hash + if ($currentHash -ne $file.sha256) { + Write-Host "LOCALLY EDITED: $($file.vendored) no longer matches its recorded hash" -ForegroundColor Red + Write-Host " recorded $($file.sha256)" + Write-Host " actual $currentHash" + $changed = $true + } + + if ($file.verbatim) { + $upstreamHash = (Get-FileHash $upstreamPath -Algorithm SHA256).Hash + if ($upstreamHash -eq $currentHash) { + Write-Host "up to date: $($file.vendored)" -ForegroundColor Green + continue + } + + Write-Host "UPSTREAM MOVED: $($file.vendored)" -ForegroundColor Yellow + $changed = $true + if ($Apply) { + Copy-Item $upstreamPath $vendoredPath -Force + $file.sha256 = (Get-FileHash $vendoredPath -Algorithm SHA256).Hash + Write-Host " updated to $($file.sha256)" + } + } + else { + # Fragment: compare the extracted class body, ignoring the MARS-authored header. + $fragment = Get-XorShiftFragment $upstreamPath + $vendored = [System.IO.File]::ReadAllText($vendoredPath) + $normalizedFragment = ($fragment -replace "`r`n", "`n") + $normalizedVendored = ($vendored -replace "`r`n", "`n") + if ($normalizedVendored.Contains($normalizedFragment)) { + Write-Host "up to date: $($file.vendored) (fragment matches upstream)" -ForegroundColor Green + } + else { + Write-Host "UPSTREAM MOVED: $($file.vendored) fragment differs from $($file.upstream)" -ForegroundColor Yellow + Write-Host " re-extract by hand; the surrounding header is MARS-authored" + $changed = $true + } + } +} + +if ($Apply) { + $manifest.commit = (git -C $PwizPath rev-parse HEAD).Trim() + $manifest.commitSubject = (git -C $PwizPath log -1 --format=%s).Trim() + $manifest.branch = (git -C $PwizPath rev-parse --abbrev-ref HEAD).Trim() + $manifest.syncedOn = (Get-Date -Format 'yyyy-MM-dd') + $manifest | ConvertTo-Json -Depth 6 | Set-Content $manifestPath -Encoding utf8NoBOM + Write-Host "`nRewrote $manifestPath" -ForegroundColor Cyan +} + +if ($changed -and -not $Apply) { + Write-Host "`nRe-run with -Apply to pull the upstream changes down." -ForegroundColor Cyan + exit 1 +} + +exit 0 diff --git a/dotnet/third_party/Osprey.ML/GradientBoostedTrees.cs b/dotnet/third_party/Osprey.ML/GradientBoostedTrees.cs new file mode 100644 index 0000000..4acd086 --- /dev/null +++ b/dotnet/third_party/Osprey.ML/GradientBoostedTrees.cs @@ -0,0 +1,830 @@ +/* + * Original author: Brendan MacLean , + * MacCoss Lab, Department of Genome Sciences, UW + * AI assistance: Claude Code (Claude Opus 4.8) + * + * Based on osprey (https://github.com/MacCossLab/osprey) + * by Michael J. MacCoss, MacCoss Lab, Department of Genome Sciences, UW + * + * Copyright 2026 University of Washington - Seattle, WA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Pure-managed gradient-boosted decision trees, used as a non-linear alternative to the +// linear Percolator SVM for FDR scoring (binary logistic) and as the m/z calibration +// model for MARS (squared error). +// +// Second-order (Newton) boosting with the XGBoost regularized objective +// (Chen & Guestrin 2016): per-leaf L2 (lambda) + L1 (alpha) penalties, minimum split +// gain (gamma), minimum child hessian, row/column subsampling, and shrinkage. +// Histogram split finding over quantile-binned features. No native dependencies +// (builds on net472 + net8.0). +// +// Everything except the base score and the per-round gradient is loss-agnostic: quantile +// binning, histogram split finding, the L1 soft-threshold and L2 leaf weight, subsampling +// and the flat node arrays all apply unchanged to either objective. +// +// The model output is the raw additive margin with no link function. For logistic that is +// a log-odds the caller ranks by exactly as with the SVM discriminant (target-decoy +// competition, q-values, PEP); for squared error it is the prediction itself. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace pwiz.Osprey.ML +{ + /// Loss function optimized by . + public enum GbtObjective + { + /// Binary logistic loss. Base score is the log-odds of the weighted + /// positive fraction; h = p(1-p)w, which never exceeds 0.25w. + LogisticBinary, + + /// Squared error. Base score is the weighted mean of y; g = (f-y)w and + /// h = w, so an unweighted hessian sum is exactly a sample count. + SquaredError + } + + /// Hyper-parameters for . Defaults are a + /// conservative, regularized setting matching the validated Python XGBoost run. + public sealed class GbtParams + { + /// Loss function. Defaults to the binary logistic loss used by the FDR path. + public GbtObjective Objective = GbtObjective.LogisticBinary; + public int NTrees = 200; + public int MaxDepth = 6; + public double LearningRate = 0.1; + /// Minimum summed hessian per leaf; blocks leaves that fit a handful of + /// points. NOTE that the hessian means different things under the two objectives: + /// under it is p(1-p), at most 0.25 and + /// shrinking as the model sharpens, so a summed hessian is far below the sample + /// count; under it is the sample weight, so + /// with unit weights a threshold of 1.0 means exactly one sample. Carry the + /// hyper-parameters over from the reference run rather than assuming these defaults + /// transfer between objectives. + public double MinChildWeight = 1.0; + /// Row subsample fraction per tree (stochastic boosting). + public double Subsample = 0.8; + /// Feature subsample fraction per tree. + public double ColSample = 0.8; + /// Minimum split gain (gamma) to keep a split. + public double Gamma = 0.0; + /// L2 penalty on leaf weights (lambda). + public double RegLambda = 1.0; + /// L1 penalty on leaf weights (alpha). + public double RegAlpha = 0.0; + /// Histogram bins per feature. CLAMPED to [2, 255] at the start of training, + /// because a bin index has to fit a byte. Note that XGBoost's own max_bin + /// default is 256, so a value transcribed from a Python configuration trains with 255 + /// here rather than the 256 it asks for. + public int MaxBins = 64; + /// Seed for the row/column subsampling PRNG. Drives + /// -- see the determinism note on + /// . + public ulong Seed = 42; + /// Threads used to accumulate histograms. Parallelism is applied ACROSS + /// FEATURES only, so every histogram is still summed in ascending row order by a + /// single thread and the trained model is bit-identical at any value. Defaults to + /// 1, which keeps the FDR path exactly sequential; raise it only for training sets + /// large enough that the accumulation dominates (millions of rows). + public int MaxDegreeOfParallelism = 1; + } + + /// + /// A trained ensemble reduced to plain arrays, so callers that need to persist a model + /// can serialize it without reflecting over private state. Round-trips exactly: the + /// arrays ARE the model, and rebuilds + /// a scorer that returns bit-identical margins. + /// + /// Internal nodes have Feature in [0, FeatureCount) and branch on Threshold + /// (value <= Threshold goes to Left), with Left and Right both greater than the + /// node's own index and different from each other. Leaves have Feature == -1, carry + /// -1 in BOTH Left and Right, and contribute Leaf, already scaled by the learning + /// rate. TreeRoot holds the node index each tree starts at. + /// + /// A writer that omits Left/Right for leaves rather than emitting -1 will be rejected + /// on load; those two fields are part of the contract, not an implementation detail. + /// + public sealed class GbtModelData + { + public int[] Feature; + public double[] Threshold; + public int[] Left; + public int[] Right; + public double[] Leaf; + public int[] TreeRoot; + public double BaseScore; + + /// Feature-vector width the model was trained on. Lets the load bounds-check + /// every split feature, so a corrupted index fails there rather than as an + /// index-out-of-range inside . + public int FeatureCount; + + /// Objective the model was trained under. Recorded because the node arrays + /// alone cannot distinguish a squared-error margin from a log-odds one, and feeding a + /// reloaded regression margin to q-value or PEP estimation would be silently wrong. + public GbtObjective Objective; + } + + /// + /// Gradient-boosted decision trees (Newton boosting) with L1/L2 leaf regularization. + /// Trained via for binary + /// classification or for + /// regression; scored via , which returns the raw additive + /// margin. + /// + /// DETERMINISTIC by construction, to the same standard as the linear SVM it stands + /// in for: identical input produces a bit-identical model and bit-identical scores, + /// on every target framework and at any . + /// The pieces that guarantee it: + /// + /// subsampling draws from -- the same seeded, + /// bit-exact-by-definition PRNG the rest of Osprey.ML uses -- NOT + /// System.Random, whose seeded sequence is a framework implementation detail + /// (this builds net472 AND net8.0, so a divergence there would silently train two + /// different models from one source); + /// every float accumulation (histograms, leaf gradients/hessians) runs in a fixed + /// row order. Histogram work may be spread across threads, but only ACROSS FEATURES: + /// one thread owns a feature's histogram and walks the node's rows in ascending order, + /// so no summation order can drift with the thread count; + /// row partitioning is stable, so each child sees its rows in the same relative + /// order they had in the parent; + /// split selection scans features and bins in ascending order and takes a new + /// best only on a strict improvement, so ties resolve to the lowest (feature, bin); + /// the one Array.Sort is over a primitive array read only by quantile + /// index, where equal values are interchangeable. + /// + /// Callers may train folds in parallel: each + /// call owns its PRNG and touches no shared state. is pure + /// and thread-safe. + /// + public sealed class GradientBoostedTrees + { + // Flattened node arrays across all trees. Internal node: Feature >= 0, split + // at Threshold (value <= Threshold -> Left, else Right). Leaf: Feature == -1, + // contribution == Leaf (already scaled by learning rate). + private readonly int[] _feature; + private readonly double[] _threshold; + private readonly int[] _left; + private readonly int[] _right; + private readonly double[] _leaf; + private readonly int[] _treeRoot; + private readonly double _baseScore; + private readonly int _featureCount; + private readonly GbtObjective _objective; + + private GradientBoostedTrees(int[] feature, double[] threshold, int[] left, int[] right, + double[] leaf, int[] treeRoot, double baseScore, int featureCount, GbtObjective objective) + { + _feature = feature; _threshold = threshold; _left = left; _right = right; + _leaf = leaf; _treeRoot = treeRoot; _baseScore = baseScore; + _featureCount = featureCount; _objective = objective; + } + + /// Feature-vector width this model expects. + public int FeatureCount { get { return _featureCount; } } + + /// Objective this model was trained under. A margin from + /// is a prediction, not a log-odds, and must + /// not be handed to q-value or PEP estimation. + public GbtObjective Objective { get { return _objective; } } + + /// + /// Train on (rows = samples, cols = features) with binary + /// labels: positive = target (!isDecoy), negative = decoy. Optional + /// per-sample weights. + /// + public static GradientBoostedTrees Train(double[][] x, bool[] isDecoy, GbtParams p, double[] sampleWeight = null) + { + if (isDecoy == null) + throw new ArgumentNullException(nameof(isDecoy)); + if (p == null) + throw new ArgumentNullException(nameof(p)); + + // This overload promises a log-odds margin from binary labels, and callers rank + // by it. A GbtParams instance carried over from a regression call would otherwise + // quietly fit squared error to 0/1 targets and return something that is not a + // log-odds at all, which q-value and PEP estimation downstream would not survive. + if (p.Objective != GbtObjective.LogisticBinary) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.Train: the binary-label overload requires GbtObjective.LogisticBinary, not {0}. Use the continuous-target overload for regression.", + p.Objective)); + } + + var y = new double[isDecoy.Length]; + for (int i = 0; i < isDecoy.Length; i++) + y[i] = isDecoy[i] ? 0.0 : 1.0; + return Train(x, y, p, sampleWeight); + } + + /// + /// Train on (rows = samples, cols = features) against a + /// continuous target , using the loss named by + /// . Optional per-sample weights. + /// + public static GradientBoostedTrees Train(double[][] x, double[] y, GbtParams p, double[] sampleWeight = null) + { + if (x == null) + throw new ArgumentNullException(nameof(x)); + if (p == null) + throw new ArgumentNullException(nameof(p)); + int n = x.Length; + if (n == 0) throw new ArgumentException(@"GradientBoostedTrees.Train: empty training set"); + if (y == null || y.Length != n) + throw new ArgumentException(@"GradientBoostedTrees.Train: target length must match the row count"); + + // Caught here rather than as an index-out-of-range partway through boosting, + // which would leave the caller guessing which array was the wrong length. + if (sampleWeight != null && sampleWeight.Length != n) + { + throw new ArgumentException( + @"GradientBoostedTrees.Train: sample weight length must match the row count"); + } + + // Under squared error the hessian IS the weight, with no positive floor of the + // kind the logistic branch applies. A negative weight can then drive a node's + // summed hessian to exactly -RegLambda and divide by zero in LeafValue, and one + // NaN leaf poisons every later round through the margin update. RegLambda is + // settable to 0 from the environment, so this is reachable in production. + if (sampleWeight != null) + { + for (int i = 0; i < n; i++) + { + if (double.IsNaN(sampleWeight[i]) || double.IsInfinity(sampleWeight[i]) || sampleWeight[i] < 0) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.Train: sample weight {0} is {1}; weights must be finite and non-negative", + i, sampleWeight[i])); + } + } + } + + switch (p.Objective) + { + case GbtObjective.LogisticBinary: + // The logistic gradient is sigmoid(f) - y, which only means anything for a + // y in [0, 1]. Leaving Objective at its default and passing a continuous + // target through this overload would otherwise train a finite-looking but + // meaningless model with no diagnostic at all. + for (int i = 0; i < n; i++) + { + if (double.IsNaN(y[i]) || y[i] < 0.0 || y[i] > 1.0) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.Train: target {0} is {1}; GbtObjective.LogisticBinary requires targets in [0, 1]. Set Objective to SquaredError for regression.", + i, y[i])); + } + } + + break; + + case GbtObjective.SquaredError: + for (int i = 0; i < n; i++) + { + if (double.IsNaN(y[i]) || double.IsInfinity(y[i])) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.Train: target {0} is {1}; targets must be finite", i, y[i])); + } + } + + break; + + default: + // An out-of-range cast would otherwise fall through to the logistic branch + // and train the wrong loss silently. + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.Train: unknown objective {0}", p.Objective)); + } + + int nFeat = x[0].Length; + int maxBins = Math.Max(2, Math.Min(255, p.MaxBins)); + if ((long)n * nFeat > int.MaxValue) + throw new ArgumentException(@"GradientBoostedTrees.Train: training matrix exceeds the addressable bin array size"); + + // --- 1. Quantile bin edges per feature; precompute byte bin indices --- + // Bins are stored column-major so histogram accumulation for one feature walks + // a contiguous run instead of striding across one small array per row. + var cuts = new double[nFeat][]; + var bin = new byte[n * nFeat]; + var col = new double[n]; + for (int j = 0; j < nFeat; j++) + { + for (int i = 0; i < n; i++) + { + double v = x[i][j]; + col[i] = double.IsNaN(v) || double.IsInfinity(v) ? 0.0 : v; + } + cuts[j] = QuantileCuts(col, maxBins); + var cj = cuts[j]; + int colStart = j * n; + for (int i = 0; i < n; i++) + bin[colStart + i] = (byte)BinOf(cj, x[i][j]); + } + + // --- 2. Weights and base score --- + var w = sampleWeight; + double pos = 0, tot = 0; + for (int i = 0; i < n; i++) + { + double wi = w != null ? w[i] : 1.0; + pos += y[i] * wi; tot += wi; + } + double baseScore; + if (p.Objective == GbtObjective.SquaredError) + { + baseScore = tot > 0 ? pos / tot : 0.0; + } + else + { + double frac = tot > 0 ? Math.Min(Math.Max(pos / tot, 1e-6), 1 - 1e-6) : 0.5; + baseScore = Math.Log(frac / (1 - frac)); + } + + var f = new double[n]; + for (int i = 0; i < n; i++) f[i] = baseScore; + var g = new double[n]; + var h = new double[n]; + + var rng = new XorShift64(p.Seed); + var nodesFeature = new List(); var nodesThresh = new List(); + var nodesLeft = new List(); var nodesRight = new List(); + var nodesLeaf = new List(); + var treeRoots = new List(); + + int nColUse = Math.Max(1, (int)Math.Round(nFeat * Math.Min(Math.Max(p.ColSample, 0.01), 1.0))); + var allFeat = new int[nFeat]; + for (int j = 0; j < nFeat; j++) allFeat[j] = j; + + var workspace = new TreeWorkspace(n, nColUse, maxBins, p, bin, g, h); + + // --- 3. Boosting rounds --- + for (int t = 0; t < p.NTrees; t++) + { + if (p.Objective == GbtObjective.SquaredError) + { + for (int i = 0; i < n; i++) + { + double wi = w != null ? w[i] : 1.0; + g[i] = (f[i] - y[i]) * wi; + h[i] = wi; + } + } + else + { + for (int i = 0; i < n; i++) + { + double pi = Sigmoid(f[i]); + double wi = w != null ? w[i] : 1.0; + g[i] = (pi - y[i]) * wi; + h[i] = Math.Max(pi * (1 - pi) * wi, 1e-6); + } + } + + // Row subsample (paired grouping is enforced upstream in fold assignment). + var rows = Subsample(n, p.Subsample, rng); + // Column subsample for this tree. + var feats = SampleColumns(allFeat, nColUse, rng); + + workspace.Reset(rows, feats); + int root = BuildTree(workspace, 0, rows.Length, 0, cuts, p, + nodesFeature, nodesThresh, nodesLeft, nodesRight, nodesLeaf); + treeRoots.Add(root); + + // Update margins for ALL samples with the new tree. The walk compares raw + // feature values, not bins, so a NaN feature takes the right branch here + // even though binning maps it to bin 0. + for (int i = 0; i < n; i++) + { + int node = root; + while (nodesFeature[node] >= 0) + node = x[i][nodesFeature[node]] <= nodesThresh[node] ? nodesLeft[node] : nodesRight[node]; + f[i] += nodesLeaf[node]; + } + } + + return new GradientBoostedTrees(nodesFeature.ToArray(), nodesThresh.ToArray(), + nodesLeft.ToArray(), nodesRight.ToArray(), nodesLeaf.ToArray(), + treeRoots.ToArray(), baseScore, nFeat, p.Objective); + } + + /// Raw additive margin for one feature vector: a log-odds under + /// , the prediction itself under + /// . + public double ScoreSingle(double[] x) + { + if (x == null) + throw new ArgumentNullException(nameof(x)); + + // A short vector would otherwise read past the caller's array only for whichever + // features the traversal happens to touch, so the failure would depend on the + // data rather than on the mistake. + if (x.Length < _featureCount) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.ScoreSingle: model expects {0} features, got {1}", + _featureCount, x.Length)); + } + + double f = _baseScore; + for (int t = 0; t < _treeRoot.Length; t++) + { + int node = _treeRoot[t]; + while (_feature[node] >= 0) + node = x[_feature[node]] <= _threshold[node] ? _left[node] : _right[node]; + f += _leaf[node]; + } + return f; + } + + /// Flatten this model to plain arrays for persistence. The arrays are + /// copies; mutating them does not affect this instance. + public GbtModelData ToModelData() + { + return new GbtModelData + { + Feature = (int[])_feature.Clone(), + Threshold = (double[])_threshold.Clone(), + Left = (int[])_left.Clone(), + Right = (int[])_right.Clone(), + Leaf = (double[])_leaf.Clone(), + TreeRoot = (int[])_treeRoot.Clone(), + BaseScore = _baseScore, + FeatureCount = _featureCount, + Objective = _objective + }; + } + + /// Rebuild a scorer from persisted arrays. Validates the node graph rather + /// than trusting it: a truncated or hand-edited model file would otherwise surface + /// as an index-out-of-range deep inside scoring, or worse, as silently wrong + /// scores. + public static GradientBoostedTrees FromModelData(GbtModelData data) + { + if (data == null) + throw new ArgumentNullException(nameof(data)); + if (data.Feature == null || data.Threshold == null || data.Left == null || + data.Right == null || data.Leaf == null || data.TreeRoot == null) + { + throw new ArgumentException(@"GradientBoostedTrees.FromModelData: incomplete model data"); + } + + int nodes = data.Feature.Length; + if (data.Threshold.Length != nodes || data.Left.Length != nodes || + data.Right.Length != nodes || data.Leaf.Length != nodes) + { + throw new ArgumentException(@"GradientBoostedTrees.FromModelData: node arrays must be the same length"); + } + if (data.TreeRoot.Length == 0) + throw new ArgumentException(@"GradientBoostedTrees.FromModelData: model has no trees"); + if (data.FeatureCount <= 0) + throw new ArgumentException(@"GradientBoostedTrees.FromModelData: feature count must be positive"); + + for (int i = 0; i < nodes; i++) + { + // Bounds-checking the split feature here is what keeps a corrupted index from + // surfacing as an index-out-of-range inside ScoreSingle instead. + if (data.Feature[i] >= data.FeatureCount) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.FromModelData: node {0} splits on feature {1}, outside the {2} the model was trained on", + i, data.Feature[i], data.FeatureCount)); + } + + if (data.Feature[i] < 0) + { + // A leaf owns no children. Rejecting stale indices here stops a partial + // edit from leaving a node that scores as a leaf but still points somewhere. + if (data.Left[i] != -1 || data.Right[i] != -1) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.FromModelData: leaf {0} carries a child index", i)); + } + + continue; + } + + if (data.Left[i] < 0 || data.Left[i] >= nodes || data.Right[i] < 0 || data.Right[i] >= nodes) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.FromModelData: node {0} has a child index outside the node array", i)); + } + + // BuildTree appends a node before recursing into its children, so a child + // index always exceeds its parent's and the two differ. Requiring that on load + // is what rules out a cycle: without it, corrupted data makes ScoreSingle spin + // forever instead of failing. + if (data.Left[i] <= i || data.Right[i] <= i || data.Left[i] == data.Right[i]) + { + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.FromModelData: node {0} has child indices {1} and {2} that do not both increase, which would make scoring cycle", + i, data.Left[i], data.Right[i])); + } + } + for (int t = 0; t < data.TreeRoot.Length; t++) + { + if (data.TreeRoot[t] < 0 || data.TreeRoot[t] >= nodes) + throw new ArgumentException(string.Format( + @"GradientBoostedTrees.FromModelData: tree {0} root is outside the node array", t)); + } + + return new GradientBoostedTrees((int[])data.Feature.Clone(), (double[])data.Threshold.Clone(), + (int[])data.Left.Clone(), (int[])data.Right.Clone(), (double[])data.Leaf.Clone(), + (int[])data.TreeRoot.Clone(), data.BaseScore, data.FeatureCount, data.Objective); + } + + private static double Sigmoid(double z) + { + if (z >= 0) { double e = Math.Exp(-z); return 1.0 / (1.0 + e); } + double ez = Math.Exp(z); return ez / (1.0 + ez); + } + + // Per-tree scratch that outlives the recursion: the row index permutation being + // partitioned in place, the pooled per-depth histograms, and the partition buffer. + // Pooling matters at scale. A fresh double[maxBins] pair per feature per node is + // millions of short-lived arrays on a multi-million-row training set. + private sealed class TreeWorkspace + { + // Only these two change per boosting round; everything else is fixed for the + // whole Train call and is therefore set once, in the constructor. + public int[] Rows; + public int[] Feats; + + public readonly byte[] Bin; + public readonly int RowStride; + public readonly double[] G; + public readonly double[] H; + public readonly int MaxBins; + public readonly int MaxDegreeOfParallelism; + + private readonly double[][] _gradHist; + private readonly double[][] _hessHist; + private readonly int[] _partition; + private readonly ParallelOptions _parallelOptions; + + public TreeWorkspace(int n, int nColUse, int maxBins, GbtParams p, + byte[] bin, double[] g, double[] h) + { + MaxBins = maxBins; + MaxDegreeOfParallelism = Math.Max(1, p.MaxDegreeOfParallelism); + Bin = bin; + RowStride = n; + G = g; + H = h; + _partition = new int[n]; + + // Allocated once rather than at every node. A depth-6 tree has up to 63 + // internal nodes, so per-node allocation would be ~12,600 throwaway objects + // per fold per boosting run. + _parallelOptions = MaxDegreeOfParallelism > 1 + ? new ParallelOptions { MaxDegreeOfParallelism = MaxDegreeOfParallelism } + : null; + + // One histogram buffer per depth that can still split: a node's histogram is + // dead once its split is chosen, so the two children share the next level's + // buffer in turn, and a node at MaxDepth is always a leaf and never builds one. + // Width is nColUse, not nFeat, because only the sampled columns are indexed. + int levels = Math.Max(1, p.MaxDepth); + _gradHist = new double[levels][]; + _hessHist = new double[levels][]; + for (int d = 0; d < levels; d++) + { + _gradHist[d] = new double[nColUse * maxBins]; + _hessHist[d] = new double[nColUse * maxBins]; + } + } + + public ParallelOptions ParallelOptions + { + get { return _parallelOptions; } + } + + public void Reset(int[] rows, int[] feats) + { + Rows = rows; Feats = feats; + } + + // Indexed directly: AccumulateHistograms is only reached for a node that can + // still split, so depth is always below MaxDepth. Clamping instead would let an + // out-of-contract depth quietly alias a parent's buffer rather than throw. + public double[] GradHist(int depth) + { + return _gradHist[depth]; + } + + public double[] HessHist(int depth) + { + return _hessHist[depth]; + } + + public int[] PartitionBuffer + { + get { return _partition; } + } + } + + // Recursively build one tree over rows [start, start + count) of the workspace's + // row permutation; appends nodes to the shared flat lists and returns the node + // index of this subtree's root. + private static int BuildTree(TreeWorkspace ws, int start, int count, int depth, + double[][] cuts, GbtParams p, + List nFeat, List nThr, List nLeft, List nRight, List nLeaf) + { + int maxBins = ws.MaxBins; + var rows = ws.Rows; + var g = ws.G; + var h = ws.H; + double gSum = 0, hSum = 0; + for (int r = start; r < start + count; r++) { int i = rows[r]; gSum += g[i]; hSum += h[i]; } + + bool leaf = depth >= p.MaxDepth || count < 2 || hSum < 2 * p.MinChildWeight; + int bestFeat = -1, bestBin = -1; + double bestGain = p.Gamma; // require gain strictly above gamma + if (!leaf) + { + var hg = ws.GradHist(depth); + var hh = ws.HessHist(depth); + var feats = ws.Feats; + AccumulateHistograms(ws, start, count, depth, hg, hh, feats); + + double parentTerm = gSum * gSum / (hSum + p.RegLambda); + for (int fi = 0; fi < feats.Length; fi++) + { + int j = feats[fi]; + int histStart = fi * maxBins; + double gl = 0, hl = 0; + for (int b = 0; b < maxBins - 1; b++) + { + gl += hg[histStart + b]; hl += hh[histStart + b]; + if (hl < 1e-12 && gl == 0) continue; + double gr = gSum - gl, hr = hSum - hl; + if (hl < p.MinChildWeight || hr < p.MinChildWeight) continue; + double gain = 0.5 * (gl * gl / (hl + p.RegLambda) + gr * gr / (hr + p.RegLambda) - parentTerm) - p.Gamma; + if (gain > bestGain) { bestGain = gain; bestFeat = j; bestBin = b; } + } + } + if (bestFeat < 0) leaf = true; + } + + if (leaf) + { + int idx = nFeat.Count; + nFeat.Add(-1); nThr.Add(0); nLeft.Add(-1); nRight.Add(-1); + nLeaf.Add(LeafValue(gSum, hSum, p)); + return idx; + } + + int leftCount = Partition(ws, start, count, bestFeat, bestBin); + + int self = nFeat.Count; + nFeat.Add(bestFeat); nThr.Add(cuts[bestFeat][bestBin]); nLeft.Add(-1); nRight.Add(-1); nLeaf.Add(0); + int lc = BuildTree(ws, start, leftCount, depth + 1, cuts, p, nFeat, nThr, nLeft, nRight, nLeaf); + int rc = BuildTree(ws, start + leftCount, count - leftCount, depth + 1, cuts, p, nFeat, nThr, nLeft, nRight, nLeaf); + nLeft[self] = lc; nRight[self] = rc; + return self; + } + + // Fill this depth's pooled histogram with the node's gradient and hessian sums per + // (sampled feature, bin). One thread owns a feature and walks the node's rows in + // ascending order, so the sums do not depend on the thread count. + // Smallest node worth handing to the scheduler, in row-by-feature accumulation steps. + // Node population halves at every level, so most nodes in a depth-6 tree are far too + // small to repay a parallel dispatch; the million-row cost this exists for lives in + // the handful of wide nodes near the root. + private const long PARALLEL_WORK_THRESHOLD = 1L << 16; + + private static void AccumulateHistograms(TreeWorkspace ws, int start, int count, int depth, + double[] hg, double[] hh, int[] feats) + { + int used = feats.Length * ws.MaxBins; + Array.Clear(hg, 0, used); + Array.Clear(hh, 0, used); + + // NOTE on the choice of Parallel.For over this assembly's own OspreyParallel.For: + // OspreyParallel allocates dedicated Threads per call, which is right at fold + // granularity (a handful of calls per run) but not here, where a call happens at + // every internal node. Thread creation would swamp the work it parallelizes. + // The trade-off is that this path uses the shared ThreadPool, so it must not be + // turned on inside fold-parallel training; MaxDegreeOfParallelism defaults to 1 + // and the FDR path never enters this branch. + if (ws.MaxDegreeOfParallelism <= 1 || + (long)count * feats.Length < PARALLEL_WORK_THRESHOLD) + { + for (int fi = 0; fi < feats.Length; fi++) + AccumulateFeature(ws, start, count, fi, hg, hh); + return; + } + + Parallel.For(0, feats.Length, ws.ParallelOptions, + fi => AccumulateFeature(ws, start, count, fi, hg, hh)); + } + + private static void AccumulateFeature(TreeWorkspace ws, int start, int count, int fi, + double[] hg, double[] hh) + { + var rows = ws.Rows; + var bin = ws.Bin; + var g = ws.G; + var h = ws.H; + int colStart = ws.Feats[fi] * ws.RowStride; + int histStart = fi * ws.MaxBins; + for (int r = start; r < start + count; r++) + { + int i = rows[r]; + int b = bin[colStart + i]; + hg[histStart + b] += g[i]; + hh[histStart + b] += h[i]; + } + } + + // Stable in-place partition of rows [start, start + count) around the chosen bin. + // Rows keep their relative order on both sides, so each child sees exactly the row + // sequence the previous list-building implementation produced. + private static int Partition(TreeWorkspace ws, int start, int count, int bestFeat, int bestBin) + { + var rows = ws.Rows; + var bin = ws.Bin; + var right = ws.PartitionBuffer; + int colStart = bestFeat * ws.RowStride; + + int leftCount = 0, rightCount = 0; + for (int r = start; r < start + count; r++) + { + int i = rows[r]; + if (bin[colStart + i] <= bestBin) + rows[start + leftCount++] = i; + else + right[rightCount++] = i; + } + + Array.Copy(right, 0, rows, start + leftCount, rightCount); + return leftCount; + } + + // Optimal leaf weight with L1 soft-threshold + L2 shrinkage, times learning rate. + private static double LeafValue(double g, double h, GbtParams p) + { + double num = g; + if (p.RegAlpha > 0) + num = g > p.RegAlpha ? g - p.RegAlpha : (g < -p.RegAlpha ? g + p.RegAlpha : 0.0); + return -p.LearningRate * num / (h + p.RegLambda); + } + + // cuts is length maxBins-1; bin index in [0, maxBins-1] = count of cuts < v. + private static int BinOf(double[] cuts, double v) + { + if (double.IsNaN(v)) return 0; + int lo = 0, hi = cuts.Length; + while (lo < hi) { int mid = (lo + hi) >> 1; if (cuts[mid] < v) lo = mid + 1; else hi = mid; } + return lo; + } + + private static double[] QuantileCuts(double[] values, int maxBins) + { + var sorted = (double[])values.Clone(); + Array.Sort(sorted); // Array.Sort OK: single primitive array read only by quantile INDEX to pick cut points; equal values are interchangeable, so tie order cannot affect the cuts + int nCut = maxBins - 1; + var cuts = new List(nCut); + double last = double.NegativeInfinity; + for (int k = 1; k <= nCut; k++) + { + double q = (double)k / maxBins; + int idx = (int)(q * (sorted.Length - 1)); + double c = sorted[idx]; + if (c > last) { cuts.Add(c); last = c; } // dedupe (skewed features) + } + if (cuts.Count == 0) cuts.Add(sorted[sorted.Length - 1]); // constant feature: one trivial cut + return cuts.ToArray(); + } + + private static int[] Subsample(int n, double frac, XorShift64 rng) + { + if (frac >= 0.999) { var all = new int[n]; for (int i = 0; i < n; i++) all[i] = i; return all; } + int m = Math.Max(1, (int)Math.Round(n * Math.Min(Math.Max(frac, 0.01), 1.0))); + var idx = new int[n]; for (int i = 0; i < n; i++) idx[i] = i; + for (int i = 0; i < m; i++) { int j = i + (int)(rng.Next() % (ulong)(n - i)); int tmp = idx[i]; idx[i] = idx[j]; idx[j] = tmp; } + var res = new int[m]; Array.Copy(idx, res, m); return res; + } + + private static int[] SampleColumns(int[] all, int k, XorShift64 rng) + { + if (k >= all.Length) return (int[])all.Clone(); + var idx = (int[])all.Clone(); + for (int i = 0; i < k; i++) { int j = i + (int)(rng.Next() % (ulong)(idx.Length - i)); int tmp = idx[i]; idx[i] = idx[j]; idx[j] = tmp; } + var res = new int[k]; Array.Copy(idx, res, k); Array.Sort(res); return res; // Array.Sort OK: single primitive array of DISTINCT feature indices (partial Fisher-Yates over a distinct set), so the comparator never ties + } + } +} diff --git a/dotnet/third_party/Osprey.ML/UPSTREAM.json b/dotnet/third_party/Osprey.ML/UPSTREAM.json new file mode 100644 index 0000000..d1556f3 --- /dev/null +++ b/dotnet/third_party/Osprey.ML/UPSTREAM.json @@ -0,0 +1,32 @@ +{ + "$comment": [ + "Drift guard for the vendored Osprey.ML sources. MARS.Test hashes every file listed", + "here and fails when one no longer matches, which turns an accidental local edit into", + "a visible test failure rather than a silent fork. Run scripts/sync-osprey-ml.ps1", + "against a pwiz checkout to pull upstream changes and rewrite these hashes.", + "", + "Osprey.ML remains the owner of this code. Fix bugs upstream in pwiz, then re-sync." + ], + "repository": "https://github.com/ProteoWizard/pwiz", + "branch": "Skyline/work/20260819_osprey_gbt_regression", + "commit": "6efbc3ea5d8f7f2c426bedd14fcf92d83fd7a8a7", + "commitSubject": "Addressed code review feedback on PR #4595", + "pullRequest": "https://github.com/ProteoWizard/pwiz/issues/4592", + "syncedOn": "2026-08-20", + "files": [ + { + "vendored": "GradientBoostedTrees.cs", + "upstream": "pwiz_tools/Osprey/Osprey.ML/GradientBoostedTrees.cs", + "verbatim": true, + "sha256": "9A9D12056BAA943E702F508F52209605E1A78439A8947DD5107669D0B23736A4" + }, + { + "vendored": "XorShift64.cs", + "upstream": "pwiz_tools/Osprey/Osprey.ML/LinearSvmClassifier.cs", + "verbatim": false, + "extractedType": "XorShift64", + "note": "Fragment extracted from a file that also pulls in MathNet.Numerics and Osprey.Core. Guarded semantically by a golden output sequence in MARS.Test, not by an upstream file hash.", + "sha256": "926BFF088079EA149F14B725BDEDB28A80181ED87B07A3839240AEB729DD0B50" + } + ] +} diff --git a/dotnet/third_party/Osprey.ML/XorShift64.cs b/dotnet/third_party/Osprey.ML/XorShift64.cs new file mode 100644 index 0000000..575c2df --- /dev/null +++ b/dotnet/third_party/Osprey.ML/XorShift64.cs @@ -0,0 +1,59 @@ +/* + * Original author: Brendan MacLean , + * MacCoss Lab, Department of Genome Sciences, UW + * + * Based on osprey (https://github.com/MacCossLab/osprey) + * by Michael J. MacCoss, MacCoss Lab, Department of Genome Sciences, UW + * + * Copyright 2026 University of Washington - Seattle, WA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// VENDORED FRAGMENT. Upstream this class lives inside +// pwiz_tools/Osprey/Osprey.ML/LinearSvmClassifier.cs, a file that also pulls in +// MathNet.Numerics and Osprey.Core and so cannot be vendored whole. Only the PRNG is +// reproduced here, because GradientBoostedTrees.cs depends on it for subsampling. +// +// The drift guard for this file is SEMANTIC, not textual: a PRNG's contract is the +// sequence it emits, so MARS.Test asserts the first outputs for a fixed seed against +// values taken from the upstream implementation. A reformatting upstream is harmless; a +// change to the shift constants would fail loudly, which is the case that matters. + +namespace pwiz.Osprey.ML +{ + /// + /// Deterministic xorshift64 PRNG for reproducible shuffling. + /// Matches the Rust implementation: x ^= x << 13; x ^= x >> 7; x ^= x << 17. + /// + public class XorShift64 + { + private ulong _state; + + public XorShift64(ulong seed) + { + // Ensure non-zero state + _state = seed == 0 ? 1UL : seed; + } + + public ulong Next() + { + ulong x = _state; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + _state = x; + return x; + } + } +} diff --git a/mars/__init__.py b/mars/__init__.py index 48ee4ce..81f85af 100644 --- a/mars/__init__.py +++ b/mars/__init__.py @@ -1,6 +1,7 @@ """Mars: Mass Accuracy Recalibration System for Thermo Stellar DIA data.""" -__version__ = "0.1.4" +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _package_version from mars.calibration import MzCalibrator from mars.library import Fragment, LibraryEntry, load_blib, load_diann_library, load_prism_library @@ -8,6 +9,14 @@ from mars.mzml import DIASpectrum, read_dia_spectra, write_calibrated_mzml from mars.visualization import plot_delta_mz_heatmap, plot_delta_mz_histogram +try: + # Single source of truth: the version declared in pyproject.toml and recorded in the + # installed distribution metadata. A literal here drifted to 0.1.4 while the package + # shipped as 0.1.5, and nothing caught it. + __version__ = _package_version("mars-ms") +except PackageNotFoundError: # running from a source tree that was never installed + __version__ = "0.0.0.dev0" + __all__ = [ # Library "LibraryEntry", diff --git a/mars/cli.py b/mars/cli.py index cc9cd8a..6dcb22a 100644 --- a/mars/cli.py +++ b/mars/cli.py @@ -12,6 +12,8 @@ import click +from mars import __version__ + # Configure logging logging.basicConfig( level=logging.INFO, @@ -73,7 +75,7 @@ def find_mzml_files( @click.group() -@click.version_option(version="0.1.5", prog_name="mars") +@click.version_option(version=__version__, prog_name="mars") def main(): """Mars: Mass Accuracy Recalibration System for Thermo Stellar DIA data.""" pass diff --git a/release-notes/README.md b/release-notes/README.md new file mode 100644 index 0000000..7b0d344 --- /dev/null +++ b/release-notes/README.md @@ -0,0 +1,130 @@ +# Release Notes + +This directory contains per-version release notes for MARS. + +## Versioning Scheme + +MARS uses a `YY.feature.patch` versioning convention, the same scheme as +[Skyline-PRISM](https://github.com/maccoss/skyline-prism): + +- **YY**: two-digit year (e.g. `26` for 2026) +- **feature**: incremented for each release containing new features +- **patch**: incremented for bug-fix-only releases within the same feature version + +Examples: `26.1.0` (first feature release of 2026), `26.1.1` (patch), `26.2.0` (second +feature release). + +The version lives in exactly one place, `dotnet/Directory.Build.props` (``), and +is updated only at release time, not during development. `mars --version` reads it back +out of the assembly, so there is nothing to keep in lockstep. + +> **The `0.1.x` line was the Python package.** MARS is now the C# tool, and its first +> release is `26.1.0`. The jump is deliberate: it is a switch of versioning scheme, not a +> hundred-and-some feature releases. The Python package is frozen to bug fixes, is no +> longer published to PyPI, and will be archived once the C# implementation has been used +> in earnest. Its notes (`RELEASE_NOTES_v0.1.*.md`) stay here as history. + +## File Format + +Each release gets one file, `RELEASE_NOTES_v{version}.md`. During development the +unreleased draft lives in `RELEASE_NOTES_next.md` and is renamed at release time. + +```text +release-notes/ + README.md # this file + RELEASE_NOTES_next.md # working draft for the next release + RELEASE_NOTES_v26.1.0.md + RELEASE_NOTES_v0.1.5.md # Python package history +``` + +## Writing Release Notes + +### During development + +Maintain `RELEASE_NOTES_next.md` as a working draft for the next planned version. Append +entries as features and fixes land. The file stays unversioned until the release is +finalized so the target version can change: a planned patch release becomes a feature +release the moment new functionality lands. + +### Content structure + +```markdown +# MARS v{version} Release Notes + +One-sentence summary of the release. + +## New Features + +- Grouped by area. What changed from the user's point of view, not how it was implemented. + +## Bug Fixes + +- What was wrong, what it affected, and what was fixed. + +## Performance + +- With context and numbers: "6.9 s for a 1.2 GB file", not "faster mzML handling". + +## Breaking Changes + +- Anything that requires the user to do something. Omit the section if there is nothing. +``` + +Sections can be omitted when empty. For a large release, subsections within a category are +fine; for a patch release a flat list is enough. + +> [!IMPORTANT] +> **Delete the empty headings when you rename the draft.** `RELEASE_NOTES_next.md` is +> seeded with all four headings so entries have somewhere to go during development, which +> means a renamed draft *always* arrives carrying the ones nobody filled in. Removing them +> is a step of the release, not something the draft gets right on its own. It matters here +> because this file is published verbatim as the GitHub Release description, where empty +> headings are visible to everyone reading the Releases page. + +### Style + +- Past tense: "Added", "Fixed", "Removed". +- Lead with user impact. +- Include specific numbers wherever they exist. +- Reference options by their CLI flag. +- **Flag anything that changes written output.** Corrected mzML files may already be in + downstream pipelines, and a change in what MARS writes is the one thing a reader cannot + afford to miss. + +## Release Process + +1. Finalize `RELEASE_NOTES_next.md` on the development branch. +2. Rename it: + `git mv release-notes/RELEASE_NOTES_next.md release-notes/RELEASE_NOTES_v{version}.md` +3. Update the title heading inside the file to match the version, and **delete every + section heading with no entries under it**. +4. Create a fresh `RELEASE_NOTES_next.md` seeded with the four headings. +5. Bump `` in `dotnet/Directory.Build.props` to `{version}`. +6. Commit and merge to `main`. +7. Tag: `git tag v{version}` +8. Push the tag: `git push origin v{version}` + +**Pushing the tag both builds the artifacts and creates the GitHub Release**, via +`.github/workflows/dotnet-release.yml`. Do not hand-create the Release. + +The workflow runs a preflight before building anything, so an inconsistent release fails +in seconds rather than after twenty minutes of artifacts: + +- the version in `Directory.Build.props` must equal the tag, +- `release-notes/RELEASE_NOTES_v{version}.md` must exist, +- and it must have no section heading with nothing under it. + +Step 2's rename therefore has to happen **before** tagging; the workflow resolves the path +from the tag. + +To fix the notes on a Release that already exists: + +```bash +gh release edit v{version} --notes-file release-notes/RELEASE_NOTES_v{version}.md +``` + +### Building artifacts without releasing + +`dotnet-release.yml` also accepts a manual `workflow_dispatch` with a version, which +builds all six platform artifacts and creates no Release. Useful for checking that +packaging works before committing to a tag. diff --git a/release-notes/RELEASE_NOTES_next.md b/release-notes/RELEASE_NOTES_next.md new file mode 100644 index 0000000..ccec5ed --- /dev/null +++ b/release-notes/RELEASE_NOTES_next.md @@ -0,0 +1,11 @@ +# MARS vNEXT Release Notes + +One-sentence summary of the release, written when there is something to summarise. + +## New Features + +## Bug Fixes + +## Performance + +## Breaking Changes diff --git a/release-notes/RELEASE_NOTES_v26.1.0.md b/release-notes/RELEASE_NOTES_v26.1.0.md new file mode 100644 index 0000000..b8f2dfb --- /dev/null +++ b/release-notes/RELEASE_NOTES_v26.1.0.md @@ -0,0 +1,521 @@ +# MARS v26.1.0 Release Notes + +MARS is now a self-contained, cross-platform command-line tool that reads Thermo, Bruker and +Sciex data directly, so a run can be calibrated straight off the instrument with no conversion +step. This is the first release of the C# implementation, which is MARS going forward. The +Python implementation is frozen to bug fixes and will be archived once the C# one has been used +in earnest; it is no longer published to PyPI. + +Versions from here follow `YY.feature.patch`, so this is the first feature release of +2026 rather than a continuation of the Python package's `0.1.x` line. + +## New Features + +- **`mars` CLI for Windows, Linux and macOS.** Five commands: `calibrate` (learn a + correction and write recalibrated mzML), `apply` (reuse a trained model), `qc` (report + mass accuracy without training or writing), `verify` (round-trip a file with a null + correction and check it), and `compare` (diff two mzML files on decoded values). +- **`.blib` libraries read without native code.** A managed SQLite reader written for this + purpose replaces `Microsoft.Data.Sqlite`, so that path pulls in no per-platform native + binary. The only native code in the tree comes from `Parquet.Net` (DIA-NN libraries) and + ships inside the release archives. +- **Streaming mzML.** Memory is bounded by the largest single spectrum plus the training + matrix rather than by file size, so a 4.9 GB Astral run uses the same working set as a + 1.2 GB Stellar one. +- **Byte-splicing writer.** The output is a byte-for-byte copy of the input except for the + m/z arrays actually corrected, which removes an entire class of serializer-induced + compatibility problems. +- **`mars verify`.** Applies a null correction and checks that the result decodes to + bit-identical arrays with a valid index and checksum. Run it before trusting any + corrected file; it separates a file-format problem from a model problem. +- **Versioned JSON model files**, recording the format version, MARS version, ordered + feature names, every hyperparameter, the acquisition-time offset and the training row + counts. Loading a model whose feature list does not match the extractor is a hard error. +- **`--min-training-rows`** (default 1000). MARS refuses to fit below this and exits 2, + rather than producing a model built on noise. +- **`--on-reorder`** controls what happens if a per-peak correction would break ascending + m/z order: `clamp` (default), `revert` or `allow`. Violations are counted and reported + under every mode. +- **Shared model implementation.** The gradient boosted trees come from `Osprey.ML`, so one + boosting implementation is maintained rather than one per tool. +- **A QC report with figures, as one self-contained HTML file.** `mars calibrate` writes + `mars_qc_report.html` alongside the text summary: the error distribution before and + after correction, median error across retention time and fragment m/z, permutation + importance, and a density panel with median-error trend lines for every active feature. + Everything is embedded - no scripts, no external references, nothing fetched when it is + opened - so the file can be attached to an email and read by someone who has neither the + data nor the tool. A 22-feature report is around 210 KB. `--no-html-report` skips it and + `--html-report ` moves it. +- **`mars qc` writes the figures too**, minus the ones that need a model: the error as + measured, how it varies across retention time and fragment m/z, and a panel per feature. + That is the report the decision to calibrate actually turns on, since `qc` is what you run + first. It stops short of predicting how much of the error is removable, because nothing + short of fitting a model answers that. `mars qc` also accepts `--temperature-dir` now, so + the temperature panels appear there as well. +- **`--dump-matches`** writes every matched fragment to CSV with all computed features, for + answering "which peak did MARS match, and what did it compute from it" without a + debugger. It is also what makes this implementation checkable against the Python one row + by row: across two Stellar runs, 160,947 matched fragments agree on all 24 shared columns + with a maximum absolute difference of zero, including the space-charge features that + carry the most weight in the model. See `docs/python-parity.md`. +- **Prebuilt binaries for six platforms**, published to GitHub Releases and self-contained + so no .NET install is needed: `win-x64`, `win-arm64`, `linux-x64`, `linux-arm64`, + `osx-arm64` and `osx-x64`, with `SHA256SUMS.txt`. Each is built and smoke tested on its + own platform where a hosted runner exists; `win-arm64` and `linux-arm64` are + cross-compiled because none is available. + +- **Documentation.** `docs/` covers the algorithm, a full CLI reference, the model in + depth, how to read the QC figures, the mzML passthrough contract, a map of the code, and + how the implementation is verified against the Python one. + +- **Cross-validation by default, with folds split by peptide.** MARS trains five models, + one per fold, and every accuracy it reports comes from a model that never saw the peptide + it is scoring. This matters because a peptide's fragments recur across hundreds of spectra + with the same theoretical m/z, and `fragment_mz` is a feature: splitting rows rather than + peptides lets the model memorize a peptide's error and report an accuracy it cannot reach + on anything new. The report gives per-fold figures, the pooled out-of-fold figure, the + spread across folds, and the gap between the two. On the reference Stellar run that gap is + 0.0014 Th, about 3% of the error being corrected, so the fit is describing the instrument + rather than the particular peptides in the run; on a data-poor window of the same cohort it + is 20%, which is a real warning that the fit is thin. `--cv-folds 0` skips cross-validation + and reports only the in-sample figure; the held-out split in that mode is also by peptide. +- **The correction model is fitted to all the data, and the report gives two numbers.** + Calibration is in-sample by nature - it is what mass calibration has always been, measuring + known species present in the run and correcting the axis from them - and the correction + moves a peak onto a fitted surface rather than onto its theoretical m/z, so there is little + scope to memorize individual peaks. The report therefore states both what the correction + achieved on these files and what cross-validation says it would achieve on a run it was not + fitted to, labelled: + + ``` + After Calibration (these files, corrected): MAD 0.0431 Th + Expected on data not used to fit: MAD 0.0445 Th + ``` + + Cross-validation costs a few extra training rounds and nothing at correction time: 66 s + against 52 s for `--cv-folds 0`, on one 1.47 GB file. +- **The QC report shows how much the folds disagree.** A per-fold table with a spread + row, two figures plotting each fold's accuracy against the pooled figure with a + one-standard-deviation band, and a plain-language reading of both the spread and the + in-sample/out-of-fold gap. One held-out number says how the model did on one split; + five say whether that number was luck. + +- **`--robust` (default `trim`) fits twice**, dropping training rows the first pass could not + explain. Matching takes the most intense peak in the tolerance window and sometimes that + peak is not the fragment; those rows carry a delta that is not a mass error, and squared + error lets them pull the fit. They are identifiable as a population - on the reference + Stellar run the 7.6% of rows with a residual beyond 0.15 Th are three times weaker, sit in + spectra with a quarter as many fragment ions, and are seven times more likely to lie + against the edge of the matching window. Removing them improves out-of-fold accuracy from + 0.0445 to 0.0442 Th, and the improvement is flat from 2 to 3 sigma, which says it is + removing a contaminant rather than tuning against the folds. Only training rows are + trimmed; held-out rows are always scored in full. `--robust none` disables the second + pass, and `--robust-sigma` moves the threshold. +- **`--robust huber`** is available and measures slightly worse, which is worth knowing: + a robust loss assumes an outlier is an extreme measurement of the right quantity, but a + mismatched peak is an accurate measurement of a different ion, and at three robust sigma + Huber still leaves such a row 79% of its weight. It is the better choice when the tail + is heavy but real rather than mislabelled. +- **Verified on high-resolution data.** One 4.9 GB Orbitrap Astral run at `--tolerance-ppm 10` + against a 16.1 GB, 67-million-row Skyline report: 1,408,902 matched fragments over 81,184 + peptides, seven minutes end to end. The correction is small and honestly reported as such - + 5.6% off the median absolute error, mostly by removing a -1.5 ppm constant offset - and + cross-validation puts the in-sample and out-of-fold figures within 0.0000 Th of each other, + so the small gain is real rather than an artifact. Pearson r is 0.144 against 0.69 on + Stellar, which is the tool correctly reporting that a well-calibrated instrument leaves + little systematic error to find. + +- **The fragment tolerance is chosen from the file.** MARS reads the mass analyzer from the + mzML's `instrumentConfiguration` and defaults to 0.3 Th on an ion trap or quadrupole and + 10 ppm on an orbitrap, FT-ICR, TOF or Astral. It says which in the log. `--resolution + unit|hram|auto` forces the choice; `--tolerance` and `--tolerance-ppm` still override + everything, because detection can be wrong on a file MARS has not seen the shape of and + the person running it can be certain in a way a heuristic cannot. + + Detection reads the analyzer for the **MS2** spectra specifically, which on a hybrid + instrument is not the run default. An Orbitrap Astral file declares an orbitrap as the run + default because that takes the MS1 survey, and points only its MS2 spectra at the Astral + analyzer. MS2 is what MARS calibrates, so that is what decides. + + This matters more than it looks, because getting it wrong is quiet. Matching the Astral + test file at the old 0.3 Th default returns 3,414,802 fragments rather than 1,408,902 - a + window about 430 ppm wide at m/z 700, filled with wrong matches - and reports a standard + deviation of 162 ppm against the 4.1 ppm really there. The run completes, writes corrected + files and produces a full report, all of it meaningless. + +- **QC reports are drawn in the units the instrument is specified in.** On high-resolution + data every axis, table and verdict is now in ppm; on trap data they stay in Th. The text + summary reports both scales side by side either way. Conversion is per row from each + fragment's own m/z, not an aggregate divided by a nominal mass - the fragments in one run + span most of a factor of four in m/z, so the shortcut would be wrong at both ends. The two + columns are therefore summaries of different per-row quantities rather than rescalings of + one another. + +- **Density figures use a viridis color scale.** The feature-versus-error panels were a + single-hue blue ramp, which has one usable dimension and spends most of it on pale values, + so the dense core and the sparse tail looked alike. They now run dark purple through green + to yellow, with a fragment-count colorbar, before and after correction side by side. Each + panel is normalized to its own busiest cell, because correcting concentrates the + distribution and a shared count scale would flatten the before panel to nearly empty; both + peaks are printed so the difference is not hidden. Both panels share one vertical range, + because the after panel being visibly tighter is the result. + + Counts map onto the ramp as a power law (`count / peak` to the 0.4, as in matplotlib's + `PowerNorm`) rather than linearly or logarithmically. Linear leaves one bright cell in a + dark field, since the core of these densities runs orders of magnitude above the tails; a + log overcorrects, putting a 500-count cell at 0.78 of the ramp against a peak of 2,854 so + that most of the core saturates and the structure inside it washes out. The power law puts + that same cell at 0.50. + +- **Titles and axis labels read as prose.** `log_intensity` renders as "log10 peak + intensity", `tic_injection_time` as "TIC x injection time", and the space-charge features + as "ions above +1 to 2 Th". The underscored names stay exact everywhere they are data - + the model file, the CSV dumps, the Python parity comparison - because they are identifiers + there. Type is larger throughout. + +- **timsTOF frames are collapsed rather than modelled.** pwiz presents an uncombined TIMS + frame as hundreds of spectra sharing one retention time and one isolation m/z, separated only + by mobility - ProteoWizard's `diaPASEF.d` is 4,631 spectra at five distinct scan times, whose + first MS2 holds two peaks. MARS asks pwiz to combine each frame's mobility scans into one + spectrum per isolation window: the same file becomes 8 MS2 across 8 isolation windows and 4 + retention times, and that first spectrum becomes 8,377 peaks. + + This is what makes Bruker data usable rather than merely tidy. MARS computes twelve of its + features from the peaks surrounding each match, and a two-peak mobility slice has no + neighbours to measure. Reading and writing both combine, so what is written matches what was + modelled. Data without an ion mobility stage is unaffected. + +- **Ion injection time earns its place feature by feature, decided over the whole run.** A trap + sets it per spectrum from its automatic gain control, so it says how full the trap was. An + instrument that accumulates for a fixed period reports the same number every time, and then + `injection_time` is a constant a tree can never split on while `tic_injection_time` is + `log_tic` rescaled - a duplicate that splits permutation importance with the feature it + duplicates. Those two are dropped when the value does not move. + + The features the injection time merely *scales* are kept. `fragment_ions`, the six + `ions_above_`/`ions_below_` windows and the six `adjacent_ratio_` features are peak sums + multiplied by it to turn an ion rate into an ion count: they need an injection time to exist, + but not to vary, because a constant multiplies them all alike and leaves every split a tree + could make still available. A run recording no injection time at all - Bruker and Sciex files + are like this - loses all fifteen, since nothing then turns a rate into a count. + + Whether it varies is decided from every matched row rather than from a sample of the head of + the run. An ion trap holds its injection time at the method's ceiling until the trap actually + fills, which on a gradient is the entire void volume, so the start of a run is the one stretch + that cannot show variation. Every Stellar file tested reads as constant there and varies well + before the end: + + | Stellar run | distinct MS2 injection times | spectra off the ceiling | first at | + |---|---|---|---| + | HeLa GPF-DIA, the reference cohort | 6,937 | 6% | spectrum 11,060 | + | HeLa standard 4 m/z DIA | 65,059 | 67% | spectrum 9,253 | + | 1 Th GPF-DIA | 2,033 | 1.8% | spectrum 8,239 | + + This decides most of the correction rather than a detail of it. On the reference cohort, + keeping the ion-population features puts the corrected MAD at 0.0464 Th against 0.0581 + without them, and the out-of-fold correlation at 0.679 against 0.513; `ions_above_0_1` alone + carries the highest permutation importance of any feature in the model. + + It also puts MARS on the same 20 features the Python implementation selects here - Python + keeps these whenever a run records an injection time, without asking whether it varies. + Running both over the cohort and scoring each one's written files with the same `mars qc`: + + | measured on the written files | uncorrected | Python | C# | + |---|---|---|---| + | MAD delta m/z | 0.0800 Th | 0.0472 Th | **0.0464 Th** | + | Std delta m/z | 0.1180 Th | 0.0882 Th | **0.0872 Th** | + | Median delta m/z | -0.0082 Th | -0.0046 Th | **-0.0025 Th** | + + Python's figures land within 0.0001 Th of the run recorded in + [the port spec](../docs/dotnet-port-spec.md), which is the control that makes the comparison + meaningful: the methodology is unchanged, so the difference is in what MARS does. + +- **Bruker and Sciex read directly too**, alongside Thermo. Bruker `.d`, `.tdf`, `.tsf` and + `.baf` on Windows and Linux; Sciex `.wiff` and `.wiff2` on Windows, which is as far as that + SDK goes. Bruker and Agilent runs are directories rather than files, and `--mzml`, + `--mzml-dir` and bare arguments all accept them. + + Verified against ProteoWizard's own vendor test files - a Bruker `diaPASEF.d`, a ZenoTOF 7600 + `.wiff2`, a SWATH `.wiff2` and a legacy `.wiff`. Bruker records no ion injection time; MARS + turns that feature group off, as it already does for an mzML without it. + +- **Thermo `.raw` read directly.** `qc`, `calibrate` and `apply` open a Thermo raw file + without a conversion step, through the pwiz-sharp vendor reader. `--mzml`, `--mzml-dir` and + bare file arguments all accept it; a directory now picks up every format MARS can read. + + It gives the same answer as the converted mzML. The same Astral run matched against the same + DIA-NN library returns 230,781 fragment matches either way, with the same median, standard + deviation and MAD to every reported digit. The mass analyzer is detected from the vendor file + as it is from an mzML, so an Astral raw picks 10 ppm and a ppm-scaled report on its own. + + Reading a raw is not faster - 53 s against 15 s for the converted mzML on that run, and + vendor reading does not thread - so what is saved is the conversion and its intermediate + file, not the read. + + mzML written from a raw is built by pwiz rather than spliced, because there is no input mzML + to copy: the passthrough guarantee covers mzML in and mzML out, and is not claimed otherwise. + + Thermo only for now. Other vendors are recognized well enough to report what is missing + rather than "unrecognized file". + +- **`--output-format mzXML`, `mzMLb` or `mgf`,** on `calibrate` and `apply`. mzML remains the + default and is still written by MARS's own byte-splice writer, which copies the input and + replaces only the m/z arrays it corrected. The other formats have no input to splice into, + so they are serialized by [pwiz-sharp](https://github.com/ProteoWizard/pwiz/pull/4178) - the + same code msconvert uses, and the code that wrote the mzML MARS reads in the first place. + + Both paths run the same correction over the same values. Writing one Stellar file both ways + and diffing with `mars compare` finds no difference: 114,021 spectra, 82,349,582 peaks, zero + m/z values differing. mzMLb is worth a look on size alone - 0.56 GB where the input was + 1.22 GB. + + The binary encoding is read from the input and matched per array. Left to its defaults pwiz + writes 64-bit *uncompressed*, which inflated a Stellar run by 61%. + + The pwiz reference is **optional**: pwiz-sharp has no package feed yet, so a MARS built + without a checkout writes mzML exactly as before and refuses the other formats with an + explanatory error. Build with `-p:PwizSharpDir=/pwiz/pwiz-sharp` to enable them. + +- **The pwiz write is parallel.** Scoring the model is where the time goes - on one Astral + run, 243 s of 308 s, against 17 s reading and about 49 s encoding - and pwiz's writers pull + spectra one at a time, so it was all landing on one core. MARS now reads a batch ahead and + corrects the batch in parallel, honouring `--threads`. That run goes from **318 s to 103 s**. + + Reads stay sequential: 5% of the work, and the vendor readers are not thread-safe. Only the + correction is parallel, and it is embarrassingly so - each spectrum is independent and + `SpectrumCorrector` holds nothing mutable. An mzXML written on 1 thread and on 12 hashes + identically. + + Worth knowing: **mzMLb is not byte-reproducible**, and not because of anything MARS does. + Two mzMLb writes of identical data at the same thread count differ, because the HDF5 + container records things that vary between writes. mzML and mzXML are reproducible. + +- **`mars --version` reports what the binary can do.** Vendor reading and the non-mzML + outputs depend on how a build was made and where it runs, and two identically named binaries + were otherwise indistinguishable until one refused a file: + + ``` + 26.1.0 + reads: .mzML, .raw, .wiff, .wiff2, .d, .tdf, .tsf, .baf + writes: mzML, mzXML, mzMLb, mgf + ``` + + It reports what is actually usable rather than what is recognized: a build without + pwiz-sharp says `.mzML` and `mzML`, an arm64 build drops Bruker, Sciex and mzMLb because + those need native x64 libraries, and `.lcd` is never advertised because no build carries a + Shimadzu reader - it is only recognized well enough to be refused with a reason. + +- **MARS warns when the matching window is far wider than the error in the data.** A tolerance + set for the wrong instrument fails silently - the window fills with peaks that are not the + fragment, and the run completes and reports numbers regardless - where one that is too narrow + fails loudly with too few matches. Only the silent direction needs catching, so after matching + MARS compares the window against the median absolute error and says so when it is more than + 50x. Trap data at its correct 0.3 Th sits around 4x, so this cannot fire on the case MARS was + built for. + + Prompted by a real case: a ZenoTOF 8600 reads correctly but reports no analyzer, because + pwiz's Sciex model table stops at the 7600. MARS then has nothing to detect from and falls + back to 0.3 Th - about 760 ppm at m/z 400 on a TOF. See `docs/open-questions.md`. + +- **Profile spectra are centroided by the vendor before use.** Sciex writes profile data - the + ZenoTOF 8600 file is 1,619 evenly spaced points in one MS2, at 0.00233 Th, which is 16 ppm at + m/z 142. MARS measures mass error by taking the most intense peak in a window, so on a sampled + curve the answer is quantised to the grid and the floor on measurable error would be several + times the error the instrument has; the twelve space-charge features would be counting + samples of one ion rather than neighbouring ions. + + pwiz exposes the vendor's own algorithm, and MARS uses it - "ABI/Analyst peak picking" here, + turning that spectrum into 210 peaks. Applied on reading and writing alike, because the model + is fitted on peak lists and correcting sampled curves with it would put every feature outside + what it saw. Only when the spectrum declares itself profile: Thermo and Bruker already deliver + centroids and are untouched. + + A corrected file written from profile input therefore comes out centroided. That is what + `msconvert --filter peakPicking` does routinely, but it is a change to the data rather than + only to the m/z values. + +## Bug Fixes + +Defects in the Python implementation that MARS does not reproduce, and defects in this +implementation caught before it shipped. Nothing in the second group ever reached a release; +they are listed because several of them changed what a corrected file contains, and because how +a tool fails is worth knowing. + +The first group came out of transcribing the Python implementation. Three of them affect files +that have already been written. + +- **`mars.__version__` reported a version that never shipped.** `mars/__init__.py` + declared `0.1.4` while `pyproject.toml` and the CLI both said `0.1.5`. All three now read + the installed distribution metadata, so there is one source of truth. Python package only. + +- **The `fileChecksum` written by the Python implementation is invalid.** It stops the SHA-1 + two bytes early, before the indentation preceding ``, where the mzML + convention is to hash up to and including the opening tag. Every mzML the Python + implementation has written fails checksum validation. The C# writer uses the correct + convention and `mars verify` checks it. +- **`absolute_time` was re-based for training but not for correction.** The Python + implementation subtracts the earliest acquisition before fitting, then feeds raw Unix + timestamps back in when writing, so every inference row landed above the largest value the + model had seen and the feature collapsed to a single branch. The offset now travels with + the model and is subtracted again at correction time. +- **The TIC features were computed from different quantities in the two paths** - the summed + intensity array when training, the `total ion current` cvParam when correcting, which + differ on Thermo centroided data. Both paths now use the summed array. +- **A `.blib` with no peak annotations produced a meaningless model.** Every peak became a + pseudo-fragment matched on its observed reference m/z, which measures the difference + between two runs' calibration errors rather than an absolute mass error. MARS now refuses + such a library and names the alternatives. Annotated peaks have their b and y m/z + recomputed from the sequence including modification deltas, where the Python + implementation recomputes from the stripped sequence and so gets modified peptides wrong. + +The two that change training - the `absolute_time` re-basing and the TIC features - are +reproducible with `--python-compat` for A/B comparison. + +- **A mistyped option stops the run instead of being ignored.** Unrecognized options were + reported as a warning *after* the command finished, so `--tolernace-ppm 10` silently + calibrated against the 0.3 Th default and `--output-dir` on `mars qc` wrote the report to the + current directory. Every command now refuses an unknown option before doing any work, + suggests the nearest real one, and reports it as an input error rather than a stack trace. + + The set of valid options is whatever the command reads, which cannot drift from the code - + and is also the trap in it, since an option resolved late has not been read when the check + runs. `--resolution` was rejected as a typo for exactly that reason; options resolved after + the check are now declared before it. Each command is tested against the options in its own + `--help` output rather than a hand-written list, and against a deliberate typo, so neither + half of that can fall behind. + +- **Cross-validation folds could be split by the wrong peptide.** A library entry that + collected no fragments is dropped as it is read, and every per-entry array shed it except + the peptide group. After the first dropped entry the groups were off by one, so fold + assignment used a neighbouring peptide's group - which is exactly the leak the grouped split + exists to prevent, and it would have reported a held-out accuracy better than the truth. No + reference library in the test set drops an entry, so the published numbers are unaffected; + a PRISM CSV whose rows all lack a product m/z for some precursor would have triggered it. + +- **`--cv-folds 0` reported no ppm figures.** The single-fit evaluation path was handed the + per-row ppm scale and ignored it, so `beforePpm` and `afterPpm` came back empty on + high-resolution data - the units that data is read in. + +- **A `.raw` and its converted mzML could disagree on acquisition time.** The pwiz adapter + parsed a timestamp with no UTC offset as machine-local where the mzML reader assumes UTC, so + the `absolute_time` feature shifted by the machine's offset. Both now use one routine. + +- **A named modification in a `.blib` no longer produces a confidently wrong fragment m/z.** + `C[Carbamidomethyl]` carries no mass, and it was dropped silently: the residue kept its + unmodified mass and every fragment past it came out wrong by the delta. With no Modifications + table to fall back on, MARS now keeps the m/z the library recorded and says how many entries + that applied to. + +- **A cohort mixing instruments is now reported.** One fragment tolerance is chosen for the + whole run from the first file's analyzer, so a directory holding both trap and + high-resolution data had one of them matched at the wrong width without saying so. The + injection-time probe now reads every file rather than the first. + +- **Applying a temperature-trained model without temperature data now warns.** The features + were substituted the way training substitutes a missing one and the run completed, with two + features pinned to a value no real spectrum produced and nothing in the output saying so. + +- **`mars verify` no longer allocates the whole file on a corrupt index offset.** The offset is + read out of the file being validated, which is exactly the file that cannot be trusted, and a + small one had the validator try to read the entire run as an index list. + +- **`mars compare` stops when the files stop lining up.** It pairs spectra by position and + checks the ids agree, which is right for comparing a file against a correction of itself, but + it cannot realign. One inserted or removed spectrum put every later pair against the wrong + spectrum and counted each as a difference, so two files differing by one spectrum reported as + differing everywhere. It now reports where alignment was lost and stops, and says the counts + cover only what preceded it. The doc comment claimed it matched by id, which it never did. + +- **`--dump-predictions` validates its array up front.** A predictions array not parallel to + the match table failed partway through writing millions of rows, leaving a half-written dump + and an index-out-of-range naming nothing. + +- **Retention time was read 60x too large from any vendor that records it in seconds.** The + pwiz adapter took the scan-start-time value and assumed minutes. Thermo writes minutes so + nothing showed; Bruker writes seconds, and a 64-minute diaPASEF run came back as 64 hours, + which would have gone into the absolute_time feature and out again as noise. Both adapters + now honour the unit the cvParam declares. Found by testing a second vendor. + +- **Numbers could have been parsed and written in the machine's locale.** MARS got its + locale-independence from `InvariantGlobalization`, which had to be relaxed for builds + carrying a vendor reader - the Thermo SDK constructs `CultureInfo("en-US")` and throws when + cultures are unavailable. Relaxing it hands `CurrentCulture` back to the operating system. + + MARS now pins the invariant culture at startup instead, so ICU is available to the SDK while + MARS's own numbers stay locale-independent. Two places that would have broken are fixed at + the source as well: `mars verify` formatted a timestamp without a culture, and - the one + that mattered - the BiblioSpec reader parsed numbers stored as SQLite text with the current + culture. Under a German locale that turns a fragment m/z of 653.835516 into 653,835,516, + with no error to show for it. + + A test class now runs the report writer, the model round-trip and the library reader under + `de-DE`, and the test host is built culture-capable so this runs in CI too - the + configuration where nobody would otherwise notice. + +- **The cross-validation gap in the HTML report had the wrong sign.** It was rendered as + in-sample minus out-of-fold, the reverse of how `CrossValidationReport.OptimismMad` defines + it, so the figure appeared negative. The text summary was always correct. + +- **`mars verify` could destroy its input.** Passing `--output` pointing at the input file + round-tripped the file onto itself and then deleted it, since `verify` removes its output + unless `--keep` is given. It now refuses when input and output resolve to the same path - + losing raw data to the one command whose purpose is to prove nothing was lost was the worst + possible failure for it to have. + +## Performance + +Measured on 16 logical cores. + +- **Null-correction round trip: 6.9 s for a 1.2 GB file** (176 MB/s), verifying 56,972,925 + peaks as bit-identical. +- **Full `calibrate` over the 5-file, 6.0 GB Stellar cohort: 229 s**, of which 13 s is + training on 282k rows by 20 features. +- **Astral plate: 369 s end to end** - 97 s to read a 16.1 GB, 67,119,180-row Skyline + report, 49 s to match each 4.7 GB run, 125 s to train on 3.37M rows. +- **Duplicate transitions collapsed.** A Skyline report lists every transition once per + replicate; collapsing the exact duplicates cut matching work roughly threefold on the + Astral plate (1,462,106 collapsed) with no effect on what the model learns. + +- **`--threads` is honoured, defaults to `auto`, and says what it chose.** On the mzML write + path the worker count was computed and then discarded, leaving concurrency bounded only by + the 512-deep read-ahead queue - `--threads 1` ran up to 16 spectra at once. Output was never + affected, since results are written in submission order and the correction is per spectrum, + but a user limiting MARS to one core on a shared machine did not get one core. + + The default was already one worker per logical processor, but nothing reported it and the + help text did not mention it. Whether the extra hardware threads of an SMT processor earn + their place is usually asserted rather than measured, so it was measured - correcting and + rewriting one 1.2 GB Stellar run on an 8-core i9-9900K with 16 logical processors, best of + two passes run in both directions: + + | threads | 2 | 4 | 6 | 8 | 10 | 12 | 16 | + |---|---|---|---|---|---|---|---| + | seconds | 150.5 | 77.4 | 52.5 | 45.4 | 42.8 | 38.7 | 36.8 | + + All 16 are 24% faster than the 8 physical cores, so the default keeps using them. No upper + ceiling is imposed: the curve is shallow past 8 and the writer's in-order drain has to become + the limit on a large enough machine, but where that falls has not been measured and a guessed + ceiling would be worse than none. A count below 1 is refused rather than silently meaning + "all of them" - `--threads $N` with `N` unset should report the mistake. + +## Breaking Changes + +- **Model files are format version 2**, adding the cross-validation summary. The version has + to match exactly - a model written by any other format version is refused rather than read on + a best guess - so a version 1 file has to be retrained. A cross-validated model file is about + five times larger, because the merged model holds every fold's trees. +- **Model files are not interchangeable with the Python implementation.** The Python model + is a pickle of an XGBoost booster; the C# model is versioned JSON. Retrain rather than + convert. +- **Corrected mzML files are not byte-identical to the Python implementation's output**, and + are not byte-identical across platforms either, because runtimes ship different zlib + builds. Decoded m/z and intensity values are identical, which is what any consumer reads. + Use `mars compare` rather than `cmp` to compare two files. + +- **`mars qc` and `mars calibrate` pick a different default tolerance on high-resolution + data.** A run that previously relied on the 0.3 Th default against Orbitrap, TOF or Astral + data will now match at 10 ppm and produce different - substantially better - numbers. Pass + `--tolerance 0.3` or `--resolution unit` to keep the old behavior. + +- **An unrecognized option is now an error (exit 1) rather than a warning.** A script passing + an option MARS does not understand will stop instead of silently continuing with defaults.