From 73b54cd48ca3a37589c896a72d92a6942a0b587b Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Sun, 10 May 2026 10:06:52 +0200 Subject: [PATCH 01/10] ci(bench): add WASM + native perf-regression tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the first automated bench-tracking infrastructure for `miden-crypto`. Two CI jobs run on every PR + every push to `next`, both feeding `benchmark-action/github-action-benchmark`: - bench-wasm: Headless-Chromium harness in `miden-bench-wasm/`. Uses `wasm-pack build --target web` + a tiny Playwright driver. Tracks RPO/RPX/Poseidon2/Blake3/Keccak `merge` + `hash_elements(100)` as compiled to wasm32+simd128 and run in V8 — the runtime real users ship in. - bench-native: `cargo bench --bench {hash,word,transpose}`. Same primitives at native speed, for comparison + native- side regression detection. Both jobs: - Store medians on the `gh-pages` branch under `bench/{wasm,native}/`, coexisting with `docs.yml`'s existing `docs/` deploy (no collision — docs.yml uses `destination_dir: docs` which only wipes that subdir). - Post a sticky PR comment with the diff vs the latest `next` baseline (one comment header per job, so wasm and native each get their own). - Alert on regression > 15 % (wasm) / 20 % (native). Don't fail the workflow — GHA Linux runners share CPUs, false-positive alerts are cheap to ignore but a failed PR check would be friction. - Upload raw results (per-batch sample distribution) as a workflow artifact for post-hoc analysis when an alert fires. Why headless Chromium and not wasmtime: V8 and Cranelift produce meaningfully different machine code from the same wasm32+simd128 input; SIMD codegen quality can differ 1.5–2x. A wasmtime number could miss real regressions that only show up in V8. For perf signals we'd actually act on, we want to be measuring what users see. Why no COOP/COEP: the bench set is single-threaded pure compute. No SharedArrayBuffer, Workers, rayon, or wasm-bindgen-rayon. Skipping COOP/COEP cuts ~3 s of CI time and removes the SW-reload flake that the wallet's prove harness routinely hits. Native bench triage: - In CI: hash, word, transpose (fast, sub-second per group) - Excluded for now (revisit): smt, merkle, partial_mt, store (moderate runtime, defer until first PR's setup is stable) - Permanently excluded: large_smt, large_smt_forest, sparse_path (multi-minute; ideal for a separate nightly job) Local validation: - `cargo check -p miden-bench-wasm --target wasm32-unknown-unknown`: clean - `wasm-pack build --release --target web`: 780 KB wasm - `node driver.mjs`: produced sane medians end-to-end (Blake3 merge ~57 ns, Keccak ~163 ns, Poseidon2 ~1.7 µs, RPX ~5.7 µs, RPO ~10.7 µs — matches the README's CPU table) - `cargo bench --bench hash` with `--output-format bencher`: produces the libtest format that `tool: cargo` parses Addresses review feedback on 0xMiden/crypto#998 from @SantiagoPittella: "we should definitely start tracking WASM performance in the E2E benchmarking infrastructure to make sure it stays that way!" --- .github/workflows/bench.yml | 195 +++++ .gitignore | 5 + Cargo.lock | 14 + Cargo.toml | 1 + miden-bench-wasm/Cargo.toml | 46 ++ miden-bench-wasm/README.md | 106 +++ miden-bench-wasm/driver.mjs | 133 ++++ miden-bench-wasm/package-lock.json | 1105 ++++++++++++++++++++++++++++ miden-bench-wasm/package.json | 14 + miden-bench-wasm/src/lib.rs | 161 ++++ miden-bench-wasm/static/bench.html | 36 + 11 files changed, 1816 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 miden-bench-wasm/Cargo.toml create mode 100644 miden-bench-wasm/README.md create mode 100644 miden-bench-wasm/driver.mjs create mode 100644 miden-bench-wasm/package-lock.json create mode 100644 miden-bench-wasm/package.json create mode 100644 miden-bench-wasm/src/lib.rs create mode 100644 miden-bench-wasm/static/bench.html diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000000..0fcf623bca --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,195 @@ +# Performance regression tracking for miden-crypto. +# +# Two jobs run on every PR + every push to `next`: +# +# 1. bench-wasm: headless-Chromium + wasm-pack harness in miden-bench-wasm/ +# tracks RPO/RPX/Poseidon2/Blake3/Keccak hash perf as +# compiled to wasm32 + run in V8 — the runtime real users +# ship in. +# +# 2. bench-native: cargo bench --bench {hash,word,transpose} on the GHA +# runner. Tracks the same primitives at native speed for +# comparison + regression detection. +# +# Both jobs feed `benchmark-action/github-action-benchmark`, which: +# - Stores each metric over time on the `gh-pages` branch under +# `bench/` (separate from the existing `docs/` subdir used by docs.yml, +# so the two don't collide). +# - Posts a sticky PR comment showing the diff vs the latest `next` +# baseline. Header per job, so wasm and native each get their own +# comment that updates in place. +# - Alerts (writes a comment header) on regression > 15 %. Doesn't fail +# the workflow — GHA Linux runners share CPUs and the noise floor on +# hash benches is real. False-positive alerts are cheap to ignore; +# false-negative reverts are expensive. +# +# This is deliberately one workflow with two jobs (rather than two +# separate workflow files): both produce sticky comments to the same PR, +# both use the same gh-pages branch, both share the same triage +# discipline. Putting them next to each other in one file keeps the +# regression-tracking story coherent. + +name: bench + +on: + pull_request: + push: + branches: [next] + workflow_dispatch: + +permissions: + contents: write # github-action-benchmark needs to push to gh-pages + pull-requests: write # for sticky PR comment + +# Skip duplicate runs on PR sync events. +concurrency: + group: bench-${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + bench-wasm: + name: WASM perf (Chromium / V8) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + wasm32 target + run: | + rustup update --no-self-update + rustup target add wasm32-unknown-unknown + + - name: Install wasm-pack + # Pinned major (`v0.13`) — wasm-pack pre-compiled binaries break + # at minor bumps occasionally. Bump deliberately, never floating. + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: "v0.13.1" + + - name: Build miden-bench-wasm + # `--target web` produces an ESM JS file that imports/instantiates + # the .wasm via `fetch()` — what the static page expects. The + # `--release` profile is required: without it the per-op cost + # is dominated by debug-mode overflow checks and the numbers are + # meaningless as a perf-tracking signal. + run: | + wasm-pack build --release --target web \ + --out-dir static/pkg miden-bench-wasm + + - name: Install Node deps + working-directory: miden-bench-wasm + run: npm install --no-audit --no-fund + + - name: Install Chromium + working-directory: miden-bench-wasm + # Chromium-only — the action's matrix support is via npx, but we + # only target one browser for now (V8 fidelity to the wallet). + run: npx playwright install --with-deps chromium + + - name: Run benches + id: run + working-directory: miden-bench-wasm + run: | + node driver.mjs > results.json + echo "results=$(cat results.json | wc -l) entries written" + # Surface the per-bench medians in the workflow summary so a + # human can eyeball them without clicking into the artifact. + { + echo '## WASM bench results' + echo '' + echo '| bench | median (ns/iter) |' + echo '|---|---|' + jq -r '.[] | "| \(.name) | \(.value | tostring) |"' results.json + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload raw results as artifact + # Keeps the full per-batch sample distribution + driver stderr + # for post-hoc analysis when an alert fires. The + # github-action-benchmark store only retains the median. + uses: actions/upload-artifact@v4 + with: + name: bench-wasm-results + path: miden-bench-wasm/results.json + retention-days: 30 + + - name: Track + alert on regression + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: customSmallerIsBetter + output-file-path: miden-bench-wasm/results.json + # Separate gh-pages subdir so this never collides with docs.yml's + # `destination_dir: docs` deploy (which lives at /docs/). + benchmark-data-dir-path: bench/wasm + gh-pages-branch: gh-pages + # Push baseline updates ONLY on push to next (not on PR runs — + # those compare against the existing baseline, they don't update it). + auto-push: ${{ github.event_name == 'push' }} + comment-on-alert: true + # Regression alert threshold. Tuned high (15 %) because GHA Linux + # runners share CPUs; 10 % would false-positive on noisy + # neighbours. Re-tighten after we have ~50 PR samples and the + # measured run-to-run variance stabilises. + alert-threshold: "115%" + fail-on-alert: false # warn via comment, don't fail the PR + summary-always: true # keep the diff comment fresh on each run + comment-always: ${{ github.event_name == 'pull_request' }} + alert-comment-cc-users: '@WiktorStarczewski' + github-token: ${{ secrets.GITHUB_TOKEN }} + + bench-native: + name: Native perf (Linux x86_64) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + run: rustup update --no-self-update + + - name: Run benches + id: run + # Initial bench triage: + # - hash: RPO/RPX/Poseidon2/Blake3/Keccak merge + sequential. + # The headline. Fast (sub-second per group). + # - word: Felt arithmetic primitives. Foundational, fast. + # - transpose: Matrix transpose. Foundational, fast. + # + # Deliberately excluded for now (revisit after first PR lands): + # - smt / merkle / partial_mt / store: moderate runtime, not the + # headline; defer until we know the wasm + tracked-native + # setup is stable. + # - large_smt / large_smt_forest / sparse_path: multi-minute + # runtimes. Wrong fit for per-PR alerts; ideal for a separate + # nightly bench job. + # - encryption / dsa: defer pending review of which sub-benches + # are stable. + run: | + cargo bench --bench hash --bench word --bench transpose \ + -p miden-crypto -- --output-format bencher 2>&1 \ + | tee bench-native.txt + + - name: Upload raw results as artifact + uses: actions/upload-artifact@v4 + with: + name: bench-native-results + path: bench-native.txt + retention-days: 30 + + - name: Track + alert on regression + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: cargo + output-file-path: bench-native.txt + benchmark-data-dir-path: bench/native + gh-pages-branch: gh-pages + auto-push: ${{ github.event_name == 'push' }} + comment-on-alert: true + # Native benches are ns-scale, so any timer noise is a larger + # relative fraction. 20 % threshold; revisit once variance data + # is in. + alert-threshold: "120%" + fail-on-alert: false + summary-always: true + comment-always: ${{ github.event_name == 'pull_request' }} + alert-comment-cc-users: '@WiktorStarczewski' + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 36683931a0..be34a51ce3 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,8 @@ cmake-build-* # Proptest tests.txt + +# wasm-pack build output for miden-bench-wasm +miden-bench-wasm/static/pkg/ +miden-bench-wasm/node_modules/ +miden-bench-wasm/results.json diff --git a/Cargo.lock b/Cargo.lock index e787ce7747..a9ed0e7b22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,11 +765,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1091,6 +1093,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "miden-bench-wasm" +version = "0.26.0" +dependencies = [ + "getrandom 0.4.2", + "miden-crypto", + "rand 0.10.1", + "rand_chacha 0.10.0", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "miden-crypto" version = "0.26.0" diff --git a/Cargo.toml b/Cargo.toml index 00a612b54a..0263a4d184 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ exclude = ["miden-crypto-fuzz"] members = [ "miden-bench", + "miden-bench-wasm", "miden-crypto", "miden-crypto-derive", "miden-field", diff --git a/miden-bench-wasm/Cargo.toml b/miden-bench-wasm/Cargo.toml new file mode 100644 index 0000000000..dd26ea5082 --- /dev/null +++ b/miden-bench-wasm/Cargo.toml @@ -0,0 +1,46 @@ +[package] +description = "WASM benchmarks for miden-crypto hash primitives, run in headless Chromium for production-fidelity perf tracking." +edition = "2024" +homepage = "https://github.com/0xMiden/crypto" +license = "MIT OR Apache-2.0" +name = "miden-bench-wasm" +publish = false +readme = "README.md" +repository = "https://github.com/0xMiden/crypto" +rust-version.workspace = true +version.workspace = true + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +# Internal — track perf of the public hash primitives. We pull `miden-crypto` +# (not just `miden-field`) because the benches measure RPO/RPX/Poseidon2/ +# Blake3/Keccak end-to-end, which live in miden-crypto's `hash` module. +# `default-features = false` strips `concurrent` (rayon — won't build for +# wasm32-unknown-unknown without wasm-bindgen-rayon scaffolding); we keep +# `std` so `once_cell` uses real `std::sync` locks rather than falling +# back to `critical-section`. The latter would require a host-provided +# `_critical_section_*` impl that the browser doesn't supply, producing +# unresolved `import "env"` references at module load. +miden-crypto = { path = "../miden-crypto", default-features = false, features = ["std"] } + +# wasm-bindgen lets us export Rust fns to JS so the Playwright driver can +# invoke them from the bench page. `web-sys` exposes `performance.now()` +# for sub-µs timing. +wasm-bindgen = "0.2" +web-sys = { version = "0.3", features = ["Performance", "Window"] } + +# `getrandom` 0.4 (pulled transitively via `rand 0.10 → getrandom`) needs +# the `wasm_js` feature on wasm32-unknown-unknown, otherwise its build +# fails with `cannot find function fill_inner in module backends`. Cargo +# unifies features within a major, but NOT across majors, so even though +# the transitive dep is 0.4 we have to declare 0.4 explicitly here to +# turn the feature on. (0.3 has the same feature, but it isn't in our +# tree and adding it wouldn't unify with the 0.4 transitive.) +getrandom = { version = "0.4", features = ["wasm_js"] } + +# Random input data for the benches. Stable seeded RNG so PR-time runs +# and baseline runs use byte-identical inputs. +rand_chacha.workspace = true +rand.workspace = true diff --git a/miden-bench-wasm/README.md b/miden-bench-wasm/README.md new file mode 100644 index 0000000000..978b758149 --- /dev/null +++ b/miden-bench-wasm/README.md @@ -0,0 +1,106 @@ +# miden-bench-wasm + +WASM benchmarks for the public hash primitives in `miden-crypto`, run in +headless Chromium so the numbers reflect what the wallet's actual users +experience (V8 JIT codegen, browser memory model) rather than what a +non-browser runtime like `wasmtime` would produce. + +## What's tracked + +Per algorithm: `merge` (2-to-1) and `hash_elements` over 100 felts. + +| Algorithm | merge bench | sequential bench | +| ----------- | ------------------------------------ | ----------------------------------------------- | +| RPO256 | `bench_rpo256_merge` | `bench_rpo256_sequential_felt_100` | +| RPX256 | `bench_rpx256_merge` | `bench_rpx256_sequential_felt_100` | +| Poseidon2 | `bench_poseidon2_merge` | `bench_poseidon2_sequential_felt_100` | +| Blake3_256 | `bench_blake3_256_merge` | `bench_blake3_256_sequential_felt_100` | +| Keccak256 | `bench_keccak256_merge` | `bench_keccak256_sequential_felt_100` | + +Each bench runs 30 batches × N iterations (N tuned per bench so each +batch is ~5–10 ms — well above `performance.now()` precision). The +driver reports the median across batches as the tracked metric; +`benchmark-action/github-action-benchmark` stores each median over time +on the `gh-pages` branch and posts a sticky PR comment with the diff vs +the latest `next` baseline. + +## Running locally + +Prereqs: Rust 1.90+, `wasm-pack`, Node 18+, Chromium auto-installed by +Playwright. + +```bash +# 1. Build the WASM lib + JS bindings into static/pkg/ +wasm-pack build --release --target web --out-dir static/pkg miden-bench-wasm + +# 2. Install JS deps (Playwright + serve) +cd miden-bench-wasm && npm install + +# 3. Run the bench (writes JSON to stdout, status to stderr) +npx playwright install chromium # first run only +node driver.mjs > /tmp/results.json +``` + +The output is: + +```json +[ + { "name": "rpo256_merge", "unit": "ns/iter", "value": 4831.2, "extra": "n=30 batch_size=2000 warmup=2000" }, + … +] +``` + +## Tuning a bench + +Per-bench `num_batches` / `batch_size` / `warmup` are in `driver.mjs`'s +`BENCH_CONFIG`. Goals: + +- `batch_size` × per-op cost ≈ **5–10 ms** per batch. Below 5 ms, timer + noise dominates. Above 10 ms, you're just paying for run length without + improving variance. +- `num_batches = 30` gives a stable median + IQR. More is over-spend; less + exposes unrelated infra noise on shared GHA runners. +- `warmup ≈ batch_size` gives V8 one full batch of un-timed iterations to + settle JIT tier-up before the timed batches begin. + +If a new bench's variance is high in CI, double `batch_size` first +(more amortization), then double `num_batches` (more samples for the +median). The 15 % alert threshold in `bench.yml` is calibrated against +this tuning — it'll need re-tuning if the runs get noisier. + +## Why headless Chromium and not wasmtime + +The wallet (and any browser-based dApp consumer) runs in V8. V8 and +Cranelift (`wasmtime`'s codegen backend) produce meaningfully different +machine code from the same `wasm32+simd128` input — SIMD codegen quality +can differ 1.5–2 ×. A `wasmtime` benchmark could cleanly miss a real +regression that only shows up in V8. For perf signals we'd actually act +on, we want to be measuring what users see. + +The cost of headless Chromium (~3 s for spinup vs `wasmtime`'s ~50 ms) +is paid once per workflow run, not per bench, so the marginal cost is +negligible. + +## Why no COOP/COEP + +The benches in scope are single-threaded pure compute. No +`SharedArrayBuffer`, no `Workers`, no rayon, no `wasm-bindgen-rayon`. +Skipping COOP/COEP cuts ~3 s from CI and removes the SW-reload flake +that the wallet's prove harness routinely hits. If a future bench needs +SAB (e.g. parallel SMT construction), we'll add COOP/COEP at that +point — the rest of the harness is unchanged. + +## Why this lives in `miden-bench-wasm/` and not `miden-crypto/benches-wasm/` + +Workspace member > sibling-of-benches. Reasons: + +- The native `cargo bench` machinery doesn't run on `wasm32-unknown-unknown` + (no `std::time::Instant`, no `std::thread`); we'd be putting our wasm + bench code right next to native benches that look identical at the + `cargo bench` invocation level but in fact require completely different + invocation tooling. Confusing. +- Workspace members have their own `Cargo.toml`, dep set, and + `crate-type = ["cdylib", "rlib"]` — a clean fit for a wasm-bindgen + target. +- `miden-bench` already exists for STARK profiling at the workspace level; + this is the symmetric setup for the wasm side. diff --git a/miden-bench-wasm/driver.mjs b/miden-bench-wasm/driver.mjs new file mode 100644 index 0000000000..d289b2c7a2 --- /dev/null +++ b/miden-bench-wasm/driver.mjs @@ -0,0 +1,133 @@ +// Headless Chromium driver for the miden-crypto WASM bench harness. +// +// Usage: node driver.mjs > results.json +// +// The script: +// 1. Spins up `npx serve` to host static/ (with pkg/ from wasm-pack). +// 2. Launches headless Chromium via Playwright. +// 3. Loads bench.html, waits for window.__bench__ to populate. +// 4. For each bench function, calls it via page.evaluate with tuned +// (num_batches, batch_size, warmup) params, collects the per-batch +// ns/iter samples, computes summary stats. +// 5. Writes a JSON array to stdout in the format the +// benchmark-action/github-action-benchmark action expects for +// `tool: customSmallerIsBetter`. +// +// Why headless Chromium and not wasmtime: the wallet's actual users run +// in V8 (Chrome/Brave/Edge), and SIMD codegen / JIT behavior differs +// meaningfully between engines. A wasmtime number wouldn't reflect what +// users see. See the PR description for the engine-fidelity argument. +// +// Why no COOP/COEP: these benches are single-threaded pure compute. No +// SharedArrayBuffer, no Workers, no rayon. Skipping COOP/COEP cuts ~3 s +// of CI time and removes the SW-reload flake the wallet's prove harness +// hits. + +import { chromium } from "playwright"; +import { spawn } from "node:child_process"; +import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const STATIC_DIR = resolve(__dirname, "static"); +const PORT = 3022; + +// Per-bench tuning: pick batch_size so each batch is ~5–10 ms (well above +// performance.now()'s ~5 µs precision in cross-origin-isolated contexts — +// or even ~100 µs in non-isolated, which is what we run in). num_batches +// is fixed at 30 so the median + IQR are stable. warmup is one batch +// worth, untimed, to settle V8's JIT. +// +// Per-op cost ranges (rough, updated as we land baseline data): +// Blake3 merge: ~100 ns/op → batch_size 100_000 +// Keccak merge: ~700 ns/op → batch_size 15_000 +// Poseidon2 merge: ~5 µs/op → batch_size 2_000 +// Rpo/Rpx merge: ~5 µs/op → batch_size 2_000 +// 100-felt seq: ~10 µs/op → batch_size 1_000 +const BENCH_CONFIG = { + bench_blake3_256_merge: { num_batches: 30, batch_size: 100_000, warmup: 100_000 }, + bench_blake3_256_sequential_felt_100: { num_batches: 30, batch_size: 5_000, warmup: 5_000 }, + bench_keccak256_merge: { num_batches: 30, batch_size: 15_000, warmup: 15_000 }, + bench_keccak256_sequential_felt_100: { num_batches: 30, batch_size: 5_000, warmup: 5_000 }, + bench_poseidon2_merge: { num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_poseidon2_sequential_felt_100: { num_batches: 30, batch_size: 200, warmup: 200 }, + bench_rpo256_merge: { num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_rpo256_sequential_felt_100: { num_batches: 30, batch_size: 200, warmup: 200 }, + bench_rpx256_merge: { num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_rpx256_sequential_felt_100: { num_batches: 30, batch_size: 200, warmup: 200 }, +}; + +function median(arr) { + const sorted = [...arr].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +async function main() { + // Static-file server. `npx serve` from local node_modules — installed + // by the workflow's `npm install` step. + const serve = spawn("npx", ["serve", STATIC_DIR, "-l", String(PORT), "-L"], { + stdio: ["ignore", "pipe", "pipe"], + }); + // Drain serve's stdout to keep the pipe healthy. + serve.stdout?.on("data", () => {}); + serve.stderr?.on("data", () => {}); + + // Wait for serve to bind. + for (let i = 0; i < 50; i++) { + try { + const r = await fetch(`http://localhost:${PORT}/bench.html`); + if (r.ok) break; + } catch {} + await sleep(100); + } + + const browser = await chromium.launch({ headless: true }); + const ctx = await browser.newContext(); + const page = await ctx.newPage(); + page.on("pageerror", (err) => process.stderr.write(`[pageerror] ${err.message}\n`)); + page.on("console", (msg) => { + if (msg.type() === "error") process.stderr.write(`[browser-error] ${msg.text()}\n`); + }); + + await page.goto(`http://localhost:${PORT}/bench.html`, { waitUntil: "load" }); + await page.waitForFunction(() => window.__bench__ && Object.keys(window.__bench__).length > 0, null, { + timeout: 30_000, + }); + + const results = []; + for (const [name, cfg] of Object.entries(BENCH_CONFIG)) { + const samples = await page.evaluate( + ({ name, cfg }) => + Array.from(window.__bench__[name](cfg.num_batches, cfg.batch_size, cfg.warmup)), + { name, cfg }, + ); + + const med = median(samples); + process.stderr.write(`${name}: median=${med.toFixed(1)} ns/iter (n=${samples.length})\n`); + + // Format expected by benchmark-action/github-action-benchmark when + // `tool: customSmallerIsBetter`. Each entry is one tracked metric. + // We surface the median; sample distribution is preserved for + // post-hoc analysis via the workflow artifact (see bench.yml). + results.push({ + name: name.replace(/^bench_/, ""), + unit: "ns/iter", + value: med, + // `extra` is shown verbatim in the chart tooltip — useful when + // diagnosing variance later. + extra: `n=${samples.length} batch_size=${cfg.batch_size} warmup=${cfg.warmup}`, + }); + } + + await browser.close(); + serve.kill(); + + process.stdout.write(JSON.stringify(results, null, 2) + "\n"); +} + +main().catch((err) => { + process.stderr.write(`driver failed: ${err.stack || err.message}\n`); + process.exit(1); +}); diff --git a/miden-bench-wasm/package-lock.json b/miden-bench-wasm/package-lock.json new file mode 100644 index 0000000000..c2897dac02 --- /dev/null +++ b/miden-bench-wasm/package-lock.json @@ -0,0 +1,1105 @@ +{ + "name": "miden-bench-wasm-driver", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "miden-bench-wasm-driver", + "version": "0.0.0", + "devDependencies": { + "playwright": "^1.50.0", + "serve": "^14.2.4" + } + }, + "node_modules/@zeit/schemas": { + "version": "2.36.0", + "resolved": "https://registry.npmjs.org/@zeit/schemas/-/schemas-2.36.0.tgz", + "integrity": "sha512-7kjMwcChYEzMKjeex9ZFXkt1AyNov9R5HZtjBKVsmVpw7pa7ZtlCGvCBC2vnnXctaYN+aRI61HjIqeetZW5ROg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ajv": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.1.0" + } + }, + "node_modules/ansi-align/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-align/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/ansi-align/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-align/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/arch": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true, + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/boxen": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-7.0.0.tgz", + "integrity": "sha512-j//dBVuyacJbvW+tvZ9HuH03fZ46QcaKvvhZickZqtB271DxJ7SNRSNxrV/dZX0085m7hISRZWbzWlJvx/rHSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-align": "^3.0.1", + "camelcase": "^7.0.0", + "chalk": "^5.0.1", + "cli-boxes": "^3.0.0", + "string-width": "^5.1.2", + "type-fest": "^2.13.0", + "widest-line": "^4.0.1", + "wrap-ansi": "^8.0.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/camelcase": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-7.0.1.tgz", + "integrity": "sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.1.tgz", + "integrity": "sha512-Fo07WOYGqMfCWHOzSXOt2CxDbC6skS/jO9ynEcmpANMoPrD+W1r1K6Vx7iNm+AQmETU1Xr2t+n8nzkV9t6xh3w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cli-boxes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-3.0.0.tgz", + "integrity": "sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/clipboardy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/clipboardy/-/clipboardy-3.0.0.tgz", + "integrity": "sha512-Su+uU5sr1jkUy1sGRpLKjKrvEOVXgSgiSInwa/qeID6aJ07yh+5NWc3h2QfjHjBnfX4LhtFcuAWKUsJ3r+fjbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "arch": "^2.2.0", + "execa": "^5.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.1.0", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-port-reachable": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-port-reachable/-/is-port-reachable-4.0.0.tgz", + "integrity": "sha512-9UoipoxYmSk6Xy7QFgRv2HDyaysmgSG75TFQs6S+3pDM7ZhKTF/bskZV+0UlABHzKjNVhPjYCLfeZUEg1wXxig==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "~1.33.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types/node_modules/mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/on-headers": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", + "dev": true, + "license": "(WTFPL OR MIT)" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-3.3.0.tgz", + "integrity": "sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw==", + "dev": true, + "license": "MIT" + }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/registry-auth-token": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.3.2.tgz", + "integrity": "sha512-JL39c60XlzCVgNrO+qq68FoNb56w/m7JYvGR2jT5iR1xBrUA3Mfx5Twk5rqTThPmQKMWydGmq8oFtDlxfrmxnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "dev": true, + "license": "MIT", + "dependencies": { + "rc": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/serve": { + "version": "14.2.6", + "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.6.tgz", + "integrity": "sha512-QEjUSA+sD4Rotm1znR8s50YqA3kYpRGPmtd5GlFxbaL9n/FdUNbqMhxClqdditSk0LlZyA/dhud6XNRTOC9x2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@zeit/schemas": "2.36.0", + "ajv": "8.18.0", + "arg": "5.0.2", + "boxen": "7.0.0", + "chalk": "5.0.1", + "chalk-template": "0.4.0", + "clipboardy": "3.0.0", + "compression": "1.8.1", + "is-port-reachable": "4.0.0", + "serve-handler": "6.1.7", + "update-check": "1.5.4" + }, + "bin": { + "serve": "build/main.js" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/serve-handler": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.7.tgz", + "integrity": "sha512-CinAq1xWb0vR3twAv9evEU8cNWkXCb9kd5ePAHUKJBkOsUpR1wt/CvGdeca7vqumL1U5cSaeVQ6zZMxiJ3yWsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.0.0", + "content-disposition": "0.5.2", + "mime-types": "2.1.18", + "minimatch": "3.1.5", + "path-is-inside": "1.0.2", + "path-to-regexp": "3.3.0", + "range-parser": "1.2.0" + } + }, + "node_modules/serve-handler/node_modules/bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/update-check": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/update-check/-/update-check-1.5.4.tgz", + "integrity": "sha512-5YHsflzHP4t1G+8WGPlvKbJEbAJGCgw+Em+dGR1KmBUbr1J36SJBqlHLjR7oob7sco5hWHGQVcr9B2poIVDDTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "registry-auth-token": "3.3.2", + "registry-url": "3.1.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/widest-line": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-4.0.1.tgz", + "integrity": "sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^5.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + } + } +} diff --git a/miden-bench-wasm/package.json b/miden-bench-wasm/package.json new file mode 100644 index 0000000000..22933d4034 --- /dev/null +++ b/miden-bench-wasm/package.json @@ -0,0 +1,14 @@ +{ + "name": "miden-bench-wasm-driver", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Headless-Chromium driver for the miden-crypto WASM bench harness.", + "scripts": { + "bench": "node driver.mjs" + }, + "devDependencies": { + "playwright": "^1.50.0", + "serve": "^14.2.4" + } +} diff --git a/miden-bench-wasm/src/lib.rs b/miden-bench-wasm/src/lib.rs new file mode 100644 index 0000000000..34beb1b2ac --- /dev/null +++ b/miden-bench-wasm/src/lib.rs @@ -0,0 +1,161 @@ +//! WASM benchmarks for the public hash primitives. +//! +//! Each `bench_*` function is exported via `wasm-bindgen` and invoked by the +//! Playwright driver from JS. The function: +//! 1. Constructs the bench's input data (deterministic seeded RNG). +//! 2. Runs `warmup_iterations` un-timed iterations to settle the V8 JIT. +//! 3. Runs `num_batches` batches of `batch_size` iterations each, timed via +//! `performance.now()` per batch (post-Spectre, single-iter timing is +//! too coarse — batching amortizes timer noise). +//! 4. Returns a `Vec` of per-iteration times in nanoseconds, one entry +//! per batch. +//! +//! The driver computes median + IQR + p99 across the batches. +//! +//! # Iteration tuning +//! +//! Pick `batch_size` so each batch is ~5–10 ms. For RPO/RPX/Poseidon2 merge +//! ops at ~2-5 µs/op in WASM, that's ~1500–4000 iterations per batch. The +//! driver passes these from JS so we can tune without rebuilding the WASM. +//! +//! # Why single-thread + no SAB +//! +//! These are pure compute, no `SharedArrayBuffer`, no rayon, no Workers. So +//! the bench page does NOT need COOP/COEP — that's the part of the wallet's +//! prove harness that's flaky. Skipping it makes CI fast and stable. + +use miden_crypto::{ + Felt, + hash::{ + HasherExt, + blake::Blake3_256, + keccak::Keccak256, + poseidon2::Poseidon2, + rpo::Rpo256, + rpx::Rpx256, + }, +}; +use rand::{RngExt, SeedableRng}; +use rand_chacha::ChaCha20Rng; +use wasm_bindgen::prelude::*; + +// Stable seed so PR-time runs and baseline runs use byte-identical input +// data. Without this, run-to-run noise from input randomness would dwarf +// the perf signal we're trying to track. +const SEED: u64 = 0x4d_69_64_65_6e_2d_43_50; // "Miden-CP" in ASCII + +// Helpers ---------------------------------------------------------------- + +/// Generates a `[u8; 32]` from the seeded RNG. Used as input to byte-oriented +/// hashes (Blake3, Keccak) before measuring `merge`. +fn make_bytes(rng: &mut ChaCha20Rng) -> [u8; 32] { + let mut buf = [0u8; 32]; + rng.fill(&mut buf); + buf +} + +/// Generates a `Vec` of length `count` from the seeded RNG. Used as +/// input to `hash_elements` benches. Uses `Felt::new_unchecked` (no reduction +/// check) to match the existing native benches in `miden-crypto/benches/ +/// common/data.rs::generate_felt_array_random` — bench inputs are stable +/// across runs from the seed, so a once-validated input set is preserved +/// run-to-run regardless of constructor. +fn make_felts(rng: &mut ChaCha20Rng, count: usize) -> Vec { + (0..count).map(|_| Felt::new_unchecked(rng.random::())).collect() +} + +/// Run `num_batches` × `batch_size` iterations of `f`, returning per-iteration +/// nanoseconds for each batch. `warmup` un-timed iterations precede the +/// measured runs to settle V8's JIT. +fn run_batched(num_batches: u32, batch_size: u32, warmup: u32, mut f: F) -> Vec +where + F: FnMut(), +{ + let perf = web_sys::window() + .expect("no window") + .performance() + .expect("no performance"); + + for _ in 0..warmup { + f(); + } + + let mut samples = Vec::with_capacity(num_batches as usize); + for _ in 0..num_batches { + let t0 = perf.now(); + for _ in 0..batch_size { + f(); + } + let elapsed_ms = perf.now() - t0; + // Convert ms (performance.now()) → ns/iter, the unit + // benchmark-action/github-action-benchmark expects. + samples.push(elapsed_ms * 1_000_000.0 / batch_size as f64); + } + samples +} + +// Bench functions -------------------------------------------------------- + +// One macro per bench shape. The closure for input setup is a `fn`-style +// item passed by name (no closure-in-macro-arg ambiguity that way), with +// each algo's `_init` helper spelled out below. Cost vs full inlining: a +// dozen extra lines, paid once at bench setup — not in the timed loop. +macro_rules! bench_merge { + ($name:ident, $hasher:ty, $init:ident) => { + #[wasm_bindgen] + pub fn $name(num_batches: u32, batch_size: u32, warmup: u32) -> Vec { + let mut rng = ChaCha20Rng::seed_from_u64(SEED); + let input = $init(&mut rng); + run_batched(num_batches, batch_size, warmup, || { + core::hint::black_box(<$hasher>::merge(core::hint::black_box(&input))); + }) + } + }; +} + +macro_rules! bench_sequential { + ($name:ident, $hasher:ty, $count:expr) => { + #[wasm_bindgen] + pub fn $name(num_batches: u32, batch_size: u32, warmup: u32) -> Vec { + let mut rng = ChaCha20Rng::seed_from_u64(SEED); + let elements = make_felts(&mut rng, $count); + run_batched(num_batches, batch_size, warmup, || { + core::hint::black_box(<$hasher>::hash_elements(core::hint::black_box(&elements))); + }) + } + }; +} + +// Per-algo input setup: produce two algo-native digests by hashing 32 random +// bytes each. Spelled out per-algo because the digest type differs across +// hashers (Word for the algebraic ones, Digest256 for byte hashes). +fn rpo256_merge_init(rng: &mut ChaCha20Rng) -> [miden_crypto::Word; 2] { + [Rpo256::hash(&make_bytes(rng)), Rpo256::hash(&make_bytes(rng))] +} +fn rpx256_merge_init(rng: &mut ChaCha20Rng) -> [miden_crypto::Word; 2] { + [Rpx256::hash(&make_bytes(rng)), Rpx256::hash(&make_bytes(rng))] +} +fn poseidon2_merge_init(rng: &mut ChaCha20Rng) -> [miden_crypto::Word; 2] { + [Poseidon2::hash(&make_bytes(rng)), Poseidon2::hash(&make_bytes(rng))] +} +fn blake3_256_merge_init(rng: &mut ChaCha20Rng) -> [::Digest; 2] { + [Blake3_256::hash(&make_bytes(rng)), Blake3_256::hash(&make_bytes(rng))] +} +fn keccak256_merge_init(rng: &mut ChaCha20Rng) -> [::Digest; 2] { + [Keccak256::hash(&make_bytes(rng)), Keccak256::hash(&make_bytes(rng))] +} + +bench_merge!(bench_rpo256_merge, Rpo256, rpo256_merge_init); +bench_sequential!(bench_rpo256_sequential_felt_100, Rpo256, 100); + +bench_merge!(bench_rpx256_merge, Rpx256, rpx256_merge_init); +bench_sequential!(bench_rpx256_sequential_felt_100, Rpx256, 100); + +bench_merge!(bench_poseidon2_merge, Poseidon2, poseidon2_merge_init); +bench_sequential!(bench_poseidon2_sequential_felt_100, Poseidon2, 100); + +bench_merge!(bench_blake3_256_merge, Blake3_256, blake3_256_merge_init); +bench_sequential!(bench_blake3_256_sequential_felt_100, Blake3_256, 100); + +bench_merge!(bench_keccak256_merge, Keccak256, keccak256_merge_init); +bench_sequential!(bench_keccak256_sequential_felt_100, Keccak256, 100); diff --git a/miden-bench-wasm/static/bench.html b/miden-bench-wasm/static/bench.html new file mode 100644 index 0000000000..e0865229e9 --- /dev/null +++ b/miden-bench-wasm/static/bench.html @@ -0,0 +1,36 @@ + + + + + miden-crypto WASM bench + + + +

miden-crypto WASM bench

+
Loading WASM module…
+ + + From 5c7c32f9e0d2f706cdf0467fb5da559ad6517590 Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Sun, 10 May 2026 10:45:13 +0200 Subject: [PATCH 02/10] ci(bench): add packed-permutation throughput + end-to-end synthetic prove MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-layer tracking, addressing the gap that the original bench set covered only `merge` / `hash_elements` — both of which go through the scalar (WIDTH=1) fast path and would show 0% delta on the simd128 PR. Adds: Layer 2 — packed-permutation throughput: bench_rpo256_packed_permute bench_rpx256_packed_permute bench_poseidon2_packed_permute Runs permute_mut on `[::Packing; 12]`. Resolves to `Felt` (WIDTH=1, scalar) on `next` — matches the merge benches and confirms the WIDTH=1 const-folded fast path is intact. After PR #998 lands, same source resolves to `PackedFelt` (WIDTH=2), ns/iter halves. The dashboard step-down on these metrics IS the simd128 win, automatic and unambiguous. Layer 3 — end-to-end synthetic prove: bench_lifted_stark_prove_blake3 Drives miden-lifted-stark::prove_multi through the full pipeline (LDE → constraint folding → DEEP → FRI → LMCS Merkle) on a 4096- row Blake3 AIR trace. Headline regression metric: "did this PR slow down the actual prove stack?" Same arithmetic shape a real prove hits, in a CI-friendly ~5 s budget per run. PCS params calibrated for bench speed (log_blowup=1, no PoW) — perf metric, not a security parameter. Local validation: - `cargo check -p miden-bench-wasm --target wasm32-unknown-unknown`: clean - `wasm-pack build --release --target web`: 1.3 MB wasm (up from 780 KB; the prove path adds LDE/FRI/Merkle code) - `node driver.mjs` end-to-end: all 14 benches produce sane medians (Blake3 ~57 ns, RPO ~10.7 µs, packed-perm matching scalar on `next`, synthetic prove ~4.6 s/iter) Notes: - Total bench wall-clock ~30 s (10 s microbenches + ~23 s prove). Within the 15-min job timeout. - Per-phase breakdown of the synthetic prove (LDE / eval_constraints / DEEP / FRI separately) is left as a follow-up — would need a custom tracing-subscriber to capture span timings, ~100 LoC vs the value of total-prove tracking on first cut. --- Cargo.lock | 6 ++ miden-bench-wasm/Cargo.toml | 19 +++++ miden-bench-wasm/README.md | 56 +++++++++++-- miden-bench-wasm/driver.mjs | 77 +++++++++++------ miden-bench-wasm/src/lib.rs | 159 +++++++++++++++++++++++++++++++++++- 5 files changed, 283 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9ed0e7b22..49a353b93a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1099,6 +1099,12 @@ version = "0.26.0" dependencies = [ "getrandom 0.4.2", "miden-crypto", + "miden-lifted-stark", + "p3-blake3-air", + "p3-dft", + "p3-field", + "p3-matrix", + "p3-symmetric", "rand 0.10.1", "rand_chacha 0.10.0", "wasm-bindgen", diff --git a/miden-bench-wasm/Cargo.toml b/miden-bench-wasm/Cargo.toml index dd26ea5082..8824f8d4cf 100644 --- a/miden-bench-wasm/Cargo.toml +++ b/miden-bench-wasm/Cargo.toml @@ -25,6 +25,25 @@ crate-type = ["cdylib", "rlib"] # unresolved `import "env"` references at module load. miden-crypto = { path = "../miden-crypto", default-features = false, features = ["std"] } +# Lifted STARK prover for the end-to-end synthetic-prove bench. `testing` +# pulls the LiftedBlake3Air fixture + the goldilocks_blake3 config helpers +# we use to assemble a small representative prove. We do NOT enable +# `parallel` — `p3-maybe-rayon` falls back to a serial impl on wasm32, and +# wasm-bindgen-rayon scaffolding is out of scope for the bench. +miden-lifted-stark = { workspace = true, features = ["testing"] } + +# Plonky3 building blocks: `Radix2DitParallel` is the DFT impl +# `miden-lifted-stark` expects (with `parallel` off it runs serially); +# `RowMajorMatrix` is the trace container; `Field` is needed to name the +# `::Packing` associated type used by the packed-perm +# benches; `Blake3Air::generate_trace_rows` produces the synthetic +# Blake3 AIR trace used by the end-to-end prove bench. +p3-dft = { workspace = true, default-features = false } +p3-field = { workspace = true, default-features = false } +p3-matrix = { workspace = true, default-features = false } +p3-blake3-air = { workspace = true, default-features = false } +p3-symmetric = { workspace = true, default-features = false } + # wasm-bindgen lets us export Rust fns to JS so the Playwright driver can # invoke them from the bench page. `web-sys` exposes `performance.now()` # for sub-µs timing. diff --git a/miden-bench-wasm/README.md b/miden-bench-wasm/README.md index 978b758149..bf17df41ed 100644 --- a/miden-bench-wasm/README.md +++ b/miden-bench-wasm/README.md @@ -7,7 +7,9 @@ non-browser runtime like `wasmtime` would produce. ## What's tracked -Per algorithm: `merge` (2-to-1) and `hash_elements` over 100 felts. +Three layers, each catches a different class of regression: + +### 1. Hash-primitive throughput (`merge` + `hash_elements(100 felts)`) | Algorithm | merge bench | sequential bench | | ----------- | ------------------------------------ | ----------------------------------------------- | @@ -17,12 +19,52 @@ Per algorithm: `merge` (2-to-1) and `hash_elements` over 100 felts. | Blake3_256 | `bench_blake3_256_merge` | `bench_blake3_256_sequential_felt_100` | | Keccak256 | `bench_keccak256_merge` | `bench_keccak256_sequential_felt_100` | -Each bench runs 30 batches × N iterations (N tuned per bench so each -batch is ~5–10 ms — well above `performance.now()` precision). The -driver reports the median across batches as the tracked metric; -`benchmark-action/github-action-benchmark` stores each median over time -on the `gh-pages` branch and posts a sticky PR comment with the diff vs -the latest `next` baseline. +Catches general regressions in the public hash API. Goes through the +**scalar** (WIDTH=1) fast path of the trait-generic Permutation impl — +identical numbers between `next` and the simd128 PR, by design (the +const-folded WIDTH=1 path is meant to be a zero-cost no-op). + +### 2. Packed-permutation throughput + +| Algorithm | bench | +| --------- | ------------------------------------- | +| RPO256 | `bench_rpo256_packed_permute` | +| RPX256 | `bench_rpx256_packed_permute` | +| Poseidon2 | `bench_poseidon2_packed_permute` | + +Runs `permute_mut(&mut [::Packing; 12])`. Resolves to: +- **`Felt`** on `next` → WIDTH=1 → scalar perm (matches layer 1's + numbers; co-tracked as a sanity check on the const-folded fast path). +- **`PackedFelt`** on the simd128 PR (#998) → WIDTH=2 → packed perm, + ns/iter halves. + +This is the layer that **automatically shows the simd128 win** the +moment #998 lands, with no bench-source changes required. + +### 3. End-to-end synthetic prove + +| bench | shape | +| ------------------------------------- | ------------------------------------------- | +| `bench_lifted_stark_prove_blake3` | full prove of a 4096-row Blake3 AIR trace | + +Drives `miden-lifted-stark::prove_multi` through the complete pipeline +(LDE → constraint folding → DEEP composition → FRI → LMCS Merkle). +Same arithmetic shape a real prove hits, on a synthetic AIR small +enough to fit a 15-minute CI budget. PCS params are calibrated for +fast bench runs (log_blowup=1, no PoW) — this is a perf-tracking bench, +not a security parameter; only the relative number matters. + +The simd128 PR's gain on packed ext-field math should drop this number +~30-50% — that's the headline regression-tracking signal for "did this +PR slow down the prove stack?" + +### Storage + alerts + +The driver reports the median across samples as each bench's tracked +metric; `benchmark-action/github-action-benchmark` stores each median +over time on the `gh-pages` branch (under `bench/wasm/`, separate from +`docs/`) and posts a sticky PR comment with the diff vs the latest +`next` baseline. Alerts fire on regression > 15 %. ## Running locally diff --git a/miden-bench-wasm/driver.mjs b/miden-bench-wasm/driver.mjs index d289b2c7a2..bf0eff53b1 100644 --- a/miden-bench-wasm/driver.mjs +++ b/miden-bench-wasm/driver.mjs @@ -33,29 +33,48 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const STATIC_DIR = resolve(__dirname, "static"); const PORT = 3022; -// Per-bench tuning: pick batch_size so each batch is ~5–10 ms (well above -// performance.now()'s ~5 µs precision in cross-origin-isolated contexts — -// or even ~100 µs in non-isolated, which is what we run in). num_batches -// is fixed at 30 so the median + IQR are stable. warmup is one batch -// worth, untimed, to settle V8's JIT. +// Per-bench tuning. Two shapes: // -// Per-op cost ranges (rough, updated as we land baseline data): -// Blake3 merge: ~100 ns/op → batch_size 100_000 -// Keccak merge: ~700 ns/op → batch_size 15_000 -// Poseidon2 merge: ~5 µs/op → batch_size 2_000 -// Rpo/Rpx merge: ~5 µs/op → batch_size 2_000 -// 100-felt seq: ~10 µs/op → batch_size 1_000 +// "batched" → (num_batches, batch_size, warmup): the function runs +// num_batches batches of batch_size iterations each, with +// `warmup` un-timed iterations preceding. Returns one +// ns/iter sample per batch. Used by all microbenches. +// Pick batch_size so each batch is ~5–10 ms (well above +// `performance.now()`'s ~5 µs precision in cross-origin- +// isolated contexts; or ~100 µs without isolation, which +// is what we run in). num_batches=30 gives a stable median +// + IQR. +// +// "runs" → (num_runs, log_n): the function runs num_runs full +// proves of a 2^log_n-row trace. Returns one ns/run +// sample per run. Used by the end-to-end synthetic-prove +// bench, where each "iter" is a full prove (~1-3 s). const BENCH_CONFIG = { - bench_blake3_256_merge: { num_batches: 30, batch_size: 100_000, warmup: 100_000 }, - bench_blake3_256_sequential_felt_100: { num_batches: 30, batch_size: 5_000, warmup: 5_000 }, - bench_keccak256_merge: { num_batches: 30, batch_size: 15_000, warmup: 15_000 }, - bench_keccak256_sequential_felt_100: { num_batches: 30, batch_size: 5_000, warmup: 5_000 }, - bench_poseidon2_merge: { num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_poseidon2_sequential_felt_100: { num_batches: 30, batch_size: 200, warmup: 200 }, - bench_rpo256_merge: { num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_rpo256_sequential_felt_100: { num_batches: 30, batch_size: 200, warmup: 200 }, - bench_rpx256_merge: { num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_rpx256_sequential_felt_100: { num_batches: 30, batch_size: 200, warmup: 200 }, + // Public-API hash-primitive throughput (scalar fast-path; same numbers + // on `next` and on the simd128 PR — these track general regressions, + // NOT the simd128 win specifically). + bench_blake3_256_merge: { shape: "batched", num_batches: 30, batch_size: 100_000, warmup: 100_000 }, + bench_blake3_256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 5_000, warmup: 5_000 }, + bench_keccak256_merge: { shape: "batched", num_batches: 30, batch_size: 15_000, warmup: 15_000 }, + bench_keccak256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 5_000, warmup: 5_000 }, + bench_poseidon2_merge: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_poseidon2_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 200, warmup: 200 }, + bench_rpo256_merge: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_rpo256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 200, warmup: 200 }, + bench_rpx256_merge: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_rpx256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 200, warmup: 200 }, + // Packed-permutation throughput. On `next` these go through the + // WIDTH=1 const-folded fast path (= scalar perm). After PR #998 lands, + // the same call resolves to WIDTH=2 packed perm and ns/iter halves. + // The dashboard step-down on this metric *is* the simd128 win. + bench_rpo256_packed_permute: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_rpx256_packed_permute: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + bench_poseidon2_packed_permute: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, + // End-to-end synthetic prove. log_n=12 (4096-row Blake3 AIR trace) — + // ~1-3 s per prove, gives the headline regression-tracking metric. + // `num_runs` is small to keep CI runtime under the 15-min job timeout + // while still producing enough samples for a stable median. + bench_lifted_stark_prove_blake3: { shape: "runs", num_runs: 5, log_n: 12 }, }; function median(arr) { @@ -99,8 +118,15 @@ async function main() { const results = []; for (const [name, cfg] of Object.entries(BENCH_CONFIG)) { const samples = await page.evaluate( - ({ name, cfg }) => - Array.from(window.__bench__[name](cfg.num_batches, cfg.batch_size, cfg.warmup)), + ({ name, cfg }) => { + const fn = window.__bench__[name]; + if (cfg.shape === "batched") { + return Array.from(fn(cfg.num_batches, cfg.batch_size, cfg.warmup)); + } else if (cfg.shape === "runs") { + return Array.from(fn(cfg.num_runs, cfg.log_n)); + } + throw new Error(`unknown shape: ${cfg.shape}`); + }, { name, cfg }, ); @@ -111,13 +137,16 @@ async function main() { // `tool: customSmallerIsBetter`. Each entry is one tracked metric. // We surface the median; sample distribution is preserved for // post-hoc analysis via the workflow artifact (see bench.yml). + const extra = cfg.shape === "batched" + ? `n=${samples.length} batch_size=${cfg.batch_size} warmup=${cfg.warmup}` + : `n=${samples.length} log_n=${cfg.log_n}`; results.push({ name: name.replace(/^bench_/, ""), unit: "ns/iter", value: med, // `extra` is shown verbatim in the chart tooltip — useful when // diagnosing variance later. - extra: `n=${samples.length} batch_size=${cfg.batch_size} warmup=${cfg.warmup}`, + extra, }); } diff --git a/miden-bench-wasm/src/lib.rs b/miden-bench-wasm/src/lib.rs index 34beb1b2ac..5fa028269c 100644 --- a/miden-bench-wasm/src/lib.rs +++ b/miden-bench-wasm/src/lib.rs @@ -24,17 +24,35 @@ //! the bench page does NOT need COOP/COEP — that's the part of the wallet's //! prove harness that's flaky. Skipping it makes CI fast and stable. +extern crate alloc; + use miden_crypto::{ Felt, hash::{ HasherExt, blake::Blake3_256, keccak::Keccak256, - poseidon2::Poseidon2, - rpo::Rpo256, - rpx::Rpx256, + poseidon2::{Poseidon2, Poseidon2Permutation256}, + rpo::{Rpo256, RpoPermutation256}, + rpx::{Rpx256, RpxPermutation256}, + }, +}; +// Prove-bench types live in lifted-stark's `testing::configs::Felt` (= +// `p3_goldilocks::Goldilocks`), which is *distinct* from +// `miden_crypto::Felt` (a wrapper struct around Goldilocks). Aliased here +// to make the difference explicit at call sites. +use miden_lifted_stark::{ + AirWitness, GenericStarkConfig, PcsParams, prove_multi, + testing::{ + airs::{ZeroAuxBuilder, blake3::LiftedBlake3Air}, + configs::{Felt as StarkFelt, QuadFelt as StarkQuadFelt, goldilocks_blake3}, }, }; +use p3_blake3_air::generate_trace_rows as generate_blake3_air_trace; +use p3_dft::Radix2DitParallel; +use p3_field::Field; +use p3_matrix::dense::RowMajorMatrix; +use p3_symmetric::Permutation; use rand::{RngExt, SeedableRng}; use rand_chacha::ChaCha20Rng; use wasm_bindgen::prelude::*; @@ -159,3 +177,138 @@ bench_sequential!(bench_blake3_256_sequential_felt_100, Blake3_256, 100); bench_merge!(bench_keccak256_merge, Keccak256, keccak256_merge_init); bench_sequential!(bench_keccak256_sequential_felt_100, Keccak256, 100); + +// Packed permutation throughput ----------------------------------------- +// +// These exercise the new trait-generic `impl> +// Permutation<[P; STATE_WIDTH]>` blanket added in 0xMiden/crypto#998. +// The state type is `::Packing`, which: +// - On `next` (no simd128 PR): resolves to `Felt`, WIDTH=1, scalar perm. +// Numerically identical to the prior concrete impl — the const-folded +// fast path the PR explicitly preserves. +// - On #998: resolves to `PackedFelt`, WIDTH=2, two candidates per +// permutation invocation. Per-call work doubles in throughput. +// +// On the bench dashboard, this metric is unchanged on `next` and steps +// down ~50% the moment #998 lands — making the simd128 win automatically +// visible in CI without bench-source changes. +// +// The merge benches above don't exercise this path (they invoke +// `permute_mut(&mut [Felt; 12])` — WIDTH=1) so they would show 0% +// difference on #998. Both shapes are deliberately tracked. + +const STATE_WIDTH: usize = 12; + +macro_rules! bench_packed_permute { + ($name:ident, $perm_ty:ident) => { + #[wasm_bindgen] + pub fn $name(num_batches: u32, batch_size: u32, warmup: u32) -> Vec { + let perm = $perm_ty; + let mut rng = ChaCha20Rng::seed_from_u64(SEED); + // Build a fully-populated state of `::Packing`. + // Each lane gets a distinct value so the perm can't shortcut + // a uniform-state edge case. + let mut state = [::Packing::ZERO; STATE_WIDTH]; + for slot in &mut state { + let lane: ::Packing = + ::Packing::from(Felt::new_unchecked(rng.random::())); + *slot = lane; + } + run_batched(num_batches, batch_size, warmup, || { + let mut s = core::hint::black_box(state); + perm.permute_mut(&mut s); + core::hint::black_box(s); + }) + } + }; +} + +bench_packed_permute!(bench_rpo256_packed_permute, RpoPermutation256); +bench_packed_permute!(bench_rpx256_packed_permute, RpxPermutation256); +bench_packed_permute!(bench_poseidon2_packed_permute, Poseidon2Permutation256); + +// End-to-end synthetic prove -------------------------------------------- +// +// Proves a small `LiftedBlake3Air` instance through `miden-lifted-stark`'s +// full pipeline (LDE, constraint folding, DEEP composition, FRI, Merkle +// commits). This is the headline regression-tracking metric: it answers +// "did this PR slow down the actual prove stack?" in one number. +// +// Why Blake3 AIR specifically (vs Keccak / Poseidon2 / Miden VM AIR): +// - Smallest setup (no round-constants table to thread through). +// - Already a workspace test fixture (no new code in `miden-lifted-stark`). +// - Exercises the same arithmetic shape as a real prove: base-field +// trace, ext-field DEEP/FRI, algebraic-hash-driven LMCS Merkle. +// - Constraint density is moderate — heavy enough that the constraint- +// evaluation phase contributes meaningfully to total prove time, so +// simd128's gain on packed ext-field math will show. +// +// Why log_blowup=1 (vs miden-vm production's 3): +// - Smaller LDE → faster prove → CI-friendly. +// - This is a perf-tracking bench, not a security parameter; the +// proven-soundness number from this config is irrelevant. We just +// need the same arithmetic shape on every run. +// +// Trace size is parameterised by `log_n` so we can tune CI runtime +// without rebuilding the wasm. Default callers should pass log_n=12 +// (4096 hashes) — empirically ~1-3s per prove in WASM, gives a stable +// median across `num_runs` repetitions while staying inside the workflow's +// 15-minute timeout. + +#[wasm_bindgen] +pub fn bench_lifted_stark_prove_blake3(num_runs: u32, log_n: u32) -> Vec { + let n = 1usize << log_n; + let mut rng = ChaCha20Rng::seed_from_u64(SEED); + let inputs: alloc::vec::Vec<[u32; 24]> = (0..n) + .map(|_| { + let mut row = [0u32; 24]; + for v in &mut row { + *v = rng.random::(); + } + row + }) + .collect(); + // Trace is built over `StarkFelt` (= Goldilocks), not the wrapper + // `miden_crypto::Felt`. The prover takes Goldilocks all the way + // through; the wrapper is for the `miden_crypto::hash` public API. + let trace: RowMajorMatrix = generate_blake3_air_trace(inputs, 0); + + // PCS params calibrated for fast bench runs, NOT production security. + // log_blowup=1 gives a 2× LDE rather than miden-vm's 8×; combined + // with no PoW, this minimises wall-clock per prove while still + // exercising the full pipeline. + let pcs = PcsParams::new( + /* log_blowup */ 1, + /* log_folding_arity */ 2, + /* log_final_degree */ 7, + /* folding_pow_bits */ 0, + /* deep_pow_bits */ 0, + /* num_queries */ 27, + /* query_pow_bits */ 0, + ) + .expect("invalid PCS params"); + + let lmcs = goldilocks_blake3::test_lmcs(); + let dft = Radix2DitParallel::::default(); + let challenger_factory = goldilocks_blake3::test_challenger; + let config: GenericStarkConfig = + GenericStarkConfig::new(pcs, lmcs, dft, challenger_factory()); + + let air = LiftedBlake3Air; + let aux = ZeroAuxBuilder::dummy(); + + let perf = web_sys::window().expect("no window").performance().expect("no performance"); + let mut samples = alloc::vec::Vec::with_capacity(num_runs as usize); + for _ in 0..num_runs { + // Fresh witness + challenger per run — prove_multi consumes the + // challenger, and we want byte-identical inputs across runs. + let witness = AirWitness::new(&trace, &[], &[]); + let instances = [(&air, witness, &aux)]; + let challenger = challenger_factory(); + let t0 = perf.now(); + let _proof = prove_multi(&config, &instances, challenger).expect("prove_multi failed"); + // ms → ns/iter (one iter == one full prove) + samples.push((perf.now() - t0) * 1_000_000.0); + } + samples +} From f742e76ed7112b2b95d6673b3aaaf4b29ced1e7d Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Sun, 10 May 2026 12:49:25 +0200 Subject: [PATCH 03/10] ci(bench): tighten regression threshold to 10% + drive variance below it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adrian's reviewer-side concern was right: setting alert thresholds at 15 % wasm / 20 % native effectively normalises CI noise as acceptable and defeats the purpose. A 15 % regression IS a regression worth investigating — the right move is to drive variance below the threshold, not widen the threshold to fit measured variance. Threshold: 110 % for both wasm and native. Anything above 10 % indicates either a real regression or a tooling problem; either way, a maintainer should look. Variance reduction: - num_batches: 30 → 50. More samples → tighter median, robust to occasional outliers from shared GHA-runner CPU contention. - batch_size: bumped per-bench so each batch is ≥ 20 ms (was 5–10 ms). At 20 ms, performance.now()'s ~100 µs precision contributes ~0.5 % timer noise per batch — well under the 10 % threshold. - synthetic prove: log_n=12 + n=5 → log_n=10 + n=15. More samples in similar wall clock (~16 s vs ~23 s). The trace is still big enough that LDE / FRI / DEEP / Merkle / constraint folding all run. Documents the tighter threshold + the tooling roadmap if 10 % proves hard to hold (iai-callgrind for native instruction-count benches, larger GHA runners, CodSpeed integration, best-of-N). Local validation: same medians as the prior run within 1-2 %, n=50 IQR visibly tighter. Total bench wall clock ~40 s (microbenches: ~24 s, prove: ~16 s) — still well under the 15-min job timeout. --- .github/workflows/bench.yml | 24 ++++++++----- miden-bench-wasm/README.md | 68 +++++++++++++++++++++++++++-------- miden-bench-wasm/driver.mjs | 70 +++++++++++++++++++++++-------------- 3 files changed, 113 insertions(+), 49 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0fcf623bca..34da9d146f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -125,11 +125,14 @@ jobs: # those compare against the existing baseline, they don't update it). auto-push: ${{ github.event_name == 'push' }} comment-on-alert: true - # Regression alert threshold. Tuned high (15 %) because GHA Linux - # runners share CPUs; 10 % would false-positive on noisy - # neighbours. Re-tighten after we have ~50 PR samples and the - # measured run-to-run variance stabilises. - alert-threshold: "115%" + # Regression alert at 10 %. Run-to-run noise on GHA shared + # runners is real but accepting 15-20 % headroom would defeat + # the purpose: a 15 % regression IS a regression worth + # investigating, not the noise floor. Drive variance down + # (longer batches, more samples) to fit under this threshold + # rather than widening the threshold to fit measured noise. + # See README "Noise reduction" for the roadmap to tighten further. + alert-threshold: "110%" fail-on-alert: false # warn via comment, don't fail the PR summary-always: true # keep the diff comment fresh on each run comment-always: ${{ github.event_name == 'pull_request' }} @@ -184,10 +187,13 @@ jobs: gh-pages-branch: gh-pages auto-push: ${{ github.event_name == 'push' }} comment-on-alert: true - # Native benches are ns-scale, so any timer noise is a larger - # relative fraction. 20 % threshold; revisit once variance data - # is in. - alert-threshold: "120%" + # Same 10 % threshold as wasm — native benches are ns-scale and + # timer noise is proportionally larger, but criterion's own + # statistical pruning already handles much of the per-iteration + # variance. If 10 % is too tight in practice, the path forward + # is more iterations / `iai-callgrind` instruction-count + # benchmarking, NOT a wider threshold. + alert-threshold: "110%" fail-on-alert: false summary-always: true comment-always: ${{ github.event_name == 'pull_request' }} diff --git a/miden-bench-wasm/README.md b/miden-bench-wasm/README.md index bf17df41ed..1a30644d4c 100644 --- a/miden-bench-wasm/README.md +++ b/miden-bench-wasm/README.md @@ -95,20 +95,60 @@ The output is: ## Tuning a bench Per-bench `num_batches` / `batch_size` / `warmup` are in `driver.mjs`'s -`BENCH_CONFIG`. Goals: - -- `batch_size` × per-op cost ≈ **5–10 ms** per batch. Below 5 ms, timer - noise dominates. Above 10 ms, you're just paying for run length without - improving variance. -- `num_batches = 30` gives a stable median + IQR. More is over-spend; less - exposes unrelated infra noise on shared GHA runners. -- `warmup ≈ batch_size` gives V8 one full batch of un-timed iterations to - settle JIT tier-up before the timed batches begin. - -If a new bench's variance is high in CI, double `batch_size` first -(more amortization), then double `num_batches` (more samples for the -median). The 15 % alert threshold in `bench.yml` is calibrated against -this tuning — it'll need re-tuning if the runs get noisier. +`BENCH_CONFIG`. Goals (calibrated against the 10 % regression threshold +in `bench.yml`): + +- **`batch_size` × per-op cost ≥ 20 ms per batch.** At 20 ms, + `performance.now()`'s ~100 µs precision (in non-COOP/COEP contexts, + which is what we run in) contributes ~0.5 % per-batch timer noise — + far below the 10 % alert threshold. Going below 20 ms exposes timer + jitter; going much above is over-spend. +- **`num_batches = 50`.** Empirically gives a stable median + tight IQR + on shared GHA runners. 30 was occasionally producing outliers that + inflated single-PR variance. Bumping further has diminishing returns. +- **`warmup ≈ one batch's worth`.** Gives V8 a full batch of un-timed + iterations to settle JIT tier-up before timed batches begin. + +If a new bench's variance is high in CI, the order to escalate is: +1. Bump `num_batches` first (cheaper than bigger batches; tightens the + median directly). +2. Then bump `batch_size` (more amortization per batch — useful for the + noisiest GHA neighbours). +3. Only as a last resort, raise the `alert-threshold` in + `.github/workflows/bench.yml`. Anything above 10 % indicates the + bench tooling is too coarse, not that the regression is acceptable. + +## Noise reduction + +The 10 % regression threshold is non-negotiable: a 15 %+ regression IS +a regression worth investigating, not a "noise floor." Anything we do +to reduce variance below 10 % is a win; widening the threshold to fit +measured noise is a loss. + +What's already in place: + +- **20+ ms per batch** so timer resolution contributes < 1 % noise. +- **50-sample median** instead of mean — robust to single-batch outliers + from CPU contention. +- **Per-bench tuning** so each metric's own profile dictates iteration + count, rather than one-size-fits-all. + +Open follow-ups, in order of cost-effectiveness: + +1. **`iai-callgrind` for native benches.** Measures retired-instruction + count via valgrind, not wall-clock — fully deterministic, identical + numbers across runs of the same code. Drops native variance to ~0%. + ~30 LoC of workflow change to add as a parallel job. +2. **Larger GHA runner sizes.** `runs-on: ubuntu-latest-4-cores` (or + `buildjet-2vcpu-ubuntu-2204` for true dedicated CPUs) cuts shared- + neighbour interference. Costs $$ per minute but makes the existing + thresholds easier to hold. +3. **CodSpeed integration.** SaaS that runs benches under valgrind for + the WASM side too. Free for OSS. Replaces the headless-Chromium + harness entirely; the comment shape and PR-review UX are similar. +4. **Multi-run-per-PR with best-of-N.** Run each bench 3 times within + the same workflow run, take the best median per metric. Doubles CI + time but cuts variance ~30-50 % on the noisiest benches. ## Why headless Chromium and not wasmtime diff --git a/miden-bench-wasm/driver.mjs b/miden-bench-wasm/driver.mjs index bf0eff53b1..5769e73434 100644 --- a/miden-bench-wasm/driver.mjs +++ b/miden-bench-wasm/driver.mjs @@ -38,43 +38,61 @@ const PORT = 3022; // "batched" → (num_batches, batch_size, warmup): the function runs // num_batches batches of batch_size iterations each, with // `warmup` un-timed iterations preceding. Returns one -// ns/iter sample per batch. Used by all microbenches. -// Pick batch_size so each batch is ~5–10 ms (well above -// `performance.now()`'s ~5 µs precision in cross-origin- -// isolated contexts; or ~100 µs without isolation, which -// is what we run in). num_batches=30 gives a stable median -// + IQR. +// ns/iter sample per batch. // // "runs" → (num_runs, log_n): the function runs num_runs full // proves of a 2^log_n-row trace. Returns one ns/run -// sample per run. Used by the end-to-end synthetic-prove -// bench, where each "iter" is a full prove (~1-3 s). +// sample per run. +// +// Tuning targets — calibrated for a 10 % regression threshold (see +// bench.yml). Two knobs drive variance down: +// +// - batch_size × per-op cost ≥ 20 ms per batch. At 20 ms, +// `performance.now()`'s ~100 µs unisolated precision contributes +// ~0.5 % timer noise per batch — well under the 10 % alert +// threshold. (Going bigger doesn't help; the median is robust +// enough that 50 batches × 20 ms hits diminishing returns.) +// +// - num_batches = 50. Stable median + tight IQR; 30 was leaving +// occasional outliers on shared GHA runners. The 50 → median trim +// gets us below 5 % run-to-run variance in informal local testing +// (Apple M5 + nominal CPU contention). +// +// If a metric still triggers spurious 10 % alerts in production, the +// next move is per-bench tuning (more batches first, then bigger +// batch sizes), then `iai-callgrind` for native instruction-count +// measurement, NOT a wider alert threshold. const BENCH_CONFIG = { // Public-API hash-primitive throughput (scalar fast-path; same numbers // on `next` and on the simd128 PR — these track general regressions, // NOT the simd128 win specifically). - bench_blake3_256_merge: { shape: "batched", num_batches: 30, batch_size: 100_000, warmup: 100_000 }, - bench_blake3_256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 5_000, warmup: 5_000 }, - bench_keccak256_merge: { shape: "batched", num_batches: 30, batch_size: 15_000, warmup: 15_000 }, - bench_keccak256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 5_000, warmup: 5_000 }, - bench_poseidon2_merge: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_poseidon2_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 200, warmup: 200 }, - bench_rpo256_merge: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_rpo256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 200, warmup: 200 }, - bench_rpx256_merge: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_rpx256_sequential_felt_100: { shape: "batched", num_batches: 30, batch_size: 200, warmup: 200 }, + // Blake3 merge: ~60 ns/op → batch 400_000 (≈24 ms/batch) + // Keccak merge: ~160 ns/op → batch 150_000 (≈24 ms/batch) + // Algebraic merge:~5 µs/op → batch 5_000 (≈25 ms/batch) + // Felt-100 seq: ~25 µs/op → batch 1_000 (≈25 ms/batch) + bench_blake3_256_merge: { shape: "batched", num_batches: 50, batch_size: 400_000, warmup: 100_000 }, + bench_blake3_256_sequential_felt_100: { shape: "batched", num_batches: 50, batch_size: 30_000, warmup: 5_000 }, + bench_keccak256_merge: { shape: "batched", num_batches: 50, batch_size: 150_000, warmup: 15_000 }, + bench_keccak256_sequential_felt_100: { shape: "batched", num_batches: 50, batch_size: 25_000, warmup: 5_000 }, + bench_poseidon2_merge: { shape: "batched", num_batches: 50, batch_size: 15_000, warmup: 2_000 }, + bench_poseidon2_sequential_felt_100: { shape: "batched", num_batches: 50, batch_size: 1_500, warmup: 200 }, + bench_rpo256_merge: { shape: "batched", num_batches: 50, batch_size: 5_000, warmup: 2_000 }, + bench_rpo256_sequential_felt_100: { shape: "batched", num_batches: 50, batch_size: 500, warmup: 200 }, + bench_rpx256_merge: { shape: "batched", num_batches: 50, batch_size: 5_000, warmup: 2_000 }, + bench_rpx256_sequential_felt_100: { shape: "batched", num_batches: 50, batch_size: 1_000, warmup: 200 }, // Packed-permutation throughput. On `next` these go through the // WIDTH=1 const-folded fast path (= scalar perm). After PR #998 lands, // the same call resolves to WIDTH=2 packed perm and ns/iter halves. // The dashboard step-down on this metric *is* the simd128 win. - bench_rpo256_packed_permute: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_rpx256_packed_permute: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - bench_poseidon2_packed_permute: { shape: "batched", num_batches: 30, batch_size: 2_000, warmup: 2_000 }, - // End-to-end synthetic prove. log_n=12 (4096-row Blake3 AIR trace) — - // ~1-3 s per prove, gives the headline regression-tracking metric. - // `num_runs` is small to keep CI runtime under the 15-min job timeout - // while still producing enough samples for a stable median. - bench_lifted_stark_prove_blake3: { shape: "runs", num_runs: 5, log_n: 12 }, + bench_rpo256_packed_permute: { shape: "batched", num_batches: 50, batch_size: 5_000, warmup: 2_000 }, + bench_rpx256_packed_permute: { shape: "batched", num_batches: 50, batch_size: 5_000, warmup: 2_000 }, + bench_poseidon2_packed_permute: { shape: "batched", num_batches: 50, batch_size: 15_000, warmup: 2_000 }, + // End-to-end synthetic prove. log_n=10 (1024-row Blake3 AIR trace, + // ~1 s per prove) + n=15 runs gives a tighter median than log_n=12 + + // n=5 in similar wall clock — more samples, smaller per-sample noise + // proportion. The trace is still big enough that LDE / FRI / Merkle / + // constraint folding all run, just shorter. + bench_lifted_stark_prove_blake3: { shape: "runs", num_runs: 15, log_n: 10 }, }; function median(arr) { From 5fc04e054e39128287ffffda88416c8f58aa2adc Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Sun, 10 May 2026 13:06:31 +0200 Subject: [PATCH 04/10] ci(bench): satisfy crypto repo's CI gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rustfmt: nightly fmt run on the new sources (lifted into the repo's rustfmt.toml-driven layout — imports_granularity=Crate, etc.) - taplo: TOML formatter ran across miden-bench-wasm/Cargo.toml - workspace-lints: added `[lints] workspace = true` so the crate inherits the workspace-wide clippy denies (required by `make workspace-check`) - clippy `unused_qualifications`: replaced two `alloc::vec::Vec` references with bare `Vec` (the prelude already imports it). The `extern crate alloc` directive is no longer needed and was removed. - cargo-shear: added `[package.metadata.cargo-shear] ignored = ["getrandom"]`. `getrandom` is a direct dep purely to enable the `wasm_js` Cargo feature on the transitive 0.4 version (cargo doesn't unify features across majors); no symbol from getrandom is named in our source, so cargo-shear correctly notices that, but removing the dep would silently break the wasm32 link. - silenced two cdylib `cargo build --tests` warnings via `test = false + doctest = false` on the lib target — wasm-bindgen exports have no Rust-side test surface; they're exercised end-to-end via driver.mjs. --- miden-bench-wasm/Cargo.toml | 39 +++++++++++++++++++++------- miden-bench-wasm/src/lib.rs | 52 ++++++++++++++----------------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/miden-bench-wasm/Cargo.toml b/miden-bench-wasm/Cargo.toml index 8824f8d4cf..a05bf545b2 100644 --- a/miden-bench-wasm/Cargo.toml +++ b/miden-bench-wasm/Cargo.toml @@ -12,6 +12,25 @@ version.workspace = true [lib] crate-type = ["cdylib", "rlib"] +# No tests / doctests in this crate — it's a wasm-bindgen export surface, +# exercised end-to-end via the Playwright driver in `driver.mjs`. Setting +# both to false silences the `cargo build --tests` warning emitted on +# `cdylib` targets that have no `#[test]` items. +doctest = false +test = false + +# Inherit workspace-level lint config (clippy denies, etc.). Required by +# `make workspace-check`. +[lints] +workspace = true + +# `getrandom` is declared as a direct dep purely to enable the `wasm_js` +# Cargo feature on the transitive 0.4 version pulled in by `rand 0.10` +# (see comment on the dep itself for why this is necessary). No symbol +# from `getrandom` is named in our source, so cargo-shear flags it as +# unused — but removing it would silently re-break the wasm32 link. +[package.metadata.cargo-shear] +ignored = ["getrandom"] [dependencies] # Internal — track perf of the public hash primitives. We pull `miden-crypto` @@ -23,14 +42,14 @@ crate-type = ["cdylib", "rlib"] # back to `critical-section`. The latter would require a host-provided # `_critical_section_*` impl that the browser doesn't supply, producing # unresolved `import "env"` references at module load. -miden-crypto = { path = "../miden-crypto", default-features = false, features = ["std"] } +miden-crypto = { default-features = false, features = ["std"], path = "../miden-crypto" } # Lifted STARK prover for the end-to-end synthetic-prove bench. `testing` # pulls the LiftedBlake3Air fixture + the goldilocks_blake3 config helpers # we use to assemble a small representative prove. We do NOT enable # `parallel` — `p3-maybe-rayon` falls back to a serial impl on wasm32, and # wasm-bindgen-rayon scaffolding is out of scope for the bench. -miden-lifted-stark = { workspace = true, features = ["testing"] } +miden-lifted-stark = { features = ["testing"], workspace = true } # Plonky3 building blocks: `Radix2DitParallel` is the DFT impl # `miden-lifted-stark` expects (with `parallel` off it runs serially); @@ -38,17 +57,17 @@ miden-lifted-stark = { workspace = true, features = ["testing"] } # `::Packing` associated type used by the packed-perm # benches; `Blake3Air::generate_trace_rows` produces the synthetic # Blake3 AIR trace used by the end-to-end prove bench. -p3-dft = { workspace = true, default-features = false } -p3-field = { workspace = true, default-features = false } -p3-matrix = { workspace = true, default-features = false } -p3-blake3-air = { workspace = true, default-features = false } -p3-symmetric = { workspace = true, default-features = false } +p3-blake3-air = { default-features = false, workspace = true } +p3-dft = { default-features = false, workspace = true } +p3-field = { default-features = false, workspace = true } +p3-matrix = { default-features = false, workspace = true } +p3-symmetric = { default-features = false, workspace = true } # wasm-bindgen lets us export Rust fns to JS so the Playwright driver can # invoke them from the bench page. `web-sys` exposes `performance.now()` # for sub-µs timing. wasm-bindgen = "0.2" -web-sys = { version = "0.3", features = ["Performance", "Window"] } +web-sys = { features = ["Performance", "Window"], version = "0.3" } # `getrandom` 0.4 (pulled transitively via `rand 0.10 → getrandom`) needs # the `wasm_js` feature on wasm32-unknown-unknown, otherwise its build @@ -57,9 +76,9 @@ web-sys = { version = "0.3", features = ["Performance", "Window"] } # the transitive dep is 0.4 we have to declare 0.4 explicitly here to # turn the feature on. (0.3 has the same feature, but it isn't in our # tree and adding it wouldn't unify with the 0.4 transitive.) -getrandom = { version = "0.4", features = ["wasm_js"] } +getrandom = { features = ["wasm_js"], version = "0.4" } # Random input data for the benches. Stable seeded RNG so PR-time runs # and baseline runs use byte-identical inputs. -rand_chacha.workspace = true rand.workspace = true +rand_chacha.workspace = true diff --git a/miden-bench-wasm/src/lib.rs b/miden-bench-wasm/src/lib.rs index 5fa028269c..3c6b68a4e6 100644 --- a/miden-bench-wasm/src/lib.rs +++ b/miden-bench-wasm/src/lib.rs @@ -4,11 +4,10 @@ //! Playwright driver from JS. The function: //! 1. Constructs the bench's input data (deterministic seeded RNG). //! 2. Runs `warmup_iterations` un-timed iterations to settle the V8 JIT. -//! 3. Runs `num_batches` batches of `batch_size` iterations each, timed via -//! `performance.now()` per batch (post-Spectre, single-iter timing is -//! too coarse — batching amortizes timer noise). -//! 4. Returns a `Vec` of per-iteration times in nanoseconds, one entry -//! per batch. +//! 3. Runs `num_batches` batches of `batch_size` iterations each, timed via `performance.now()` +//! per batch (post-Spectre, single-iter timing is too coarse — batching amortizes timer +//! noise). +//! 4. Returns a `Vec` of per-iteration times in nanoseconds, one entry per batch. //! //! The driver computes median + IQR + p99 across the batches. //! @@ -24,8 +23,6 @@ //! the bench page does NOT need COOP/COEP — that's the part of the wallet's //! prove harness that's flaky. Skipping it makes CI fast and stable. -extern crate alloc; - use miden_crypto::{ Felt, hash::{ @@ -89,10 +86,7 @@ fn run_batched(num_batches: u32, batch_size: u32, warmup: u32, mut f: F) -> V where F: FnMut(), { - let perf = web_sys::window() - .expect("no window") - .performance() - .expect("no performance"); + let perf = web_sys::window().expect("no window").performance().expect("no performance"); for _ in 0..warmup { f(); @@ -183,11 +177,10 @@ bench_sequential!(bench_keccak256_sequential_felt_100, Keccak256, 100); // These exercise the new trait-generic `impl> // Permutation<[P; STATE_WIDTH]>` blanket added in 0xMiden/crypto#998. // The state type is `::Packing`, which: -// - On `next` (no simd128 PR): resolves to `Felt`, WIDTH=1, scalar perm. -// Numerically identical to the prior concrete impl — the const-folded -// fast path the PR explicitly preserves. -// - On #998: resolves to `PackedFelt`, WIDTH=2, two candidates per -// permutation invocation. Per-call work doubles in throughput. +// - On `next` (no simd128 PR): resolves to `Felt`, WIDTH=1, scalar perm. Numerically identical to +// the prior concrete impl — the const-folded fast path the PR explicitly preserves. +// - On #998: resolves to `PackedFelt`, WIDTH=2, two candidates per permutation invocation. +// Per-call work doubles in throughput. // // On the bench dashboard, this metric is unchanged on `next` and steps // down ~50% the moment #998 lands — making the simd128 win automatically @@ -237,17 +230,16 @@ bench_packed_permute!(bench_poseidon2_packed_permute, Poseidon2Permutation256); // Why Blake3 AIR specifically (vs Keccak / Poseidon2 / Miden VM AIR): // - Smallest setup (no round-constants table to thread through). // - Already a workspace test fixture (no new code in `miden-lifted-stark`). -// - Exercises the same arithmetic shape as a real prove: base-field -// trace, ext-field DEEP/FRI, algebraic-hash-driven LMCS Merkle. -// - Constraint density is moderate — heavy enough that the constraint- -// evaluation phase contributes meaningfully to total prove time, so -// simd128's gain on packed ext-field math will show. +// - Exercises the same arithmetic shape as a real prove: base-field trace, ext-field DEEP/FRI, +// algebraic-hash-driven LMCS Merkle. +// - Constraint density is moderate — heavy enough that the constraint- evaluation phase +// contributes meaningfully to total prove time, so simd128's gain on packed ext-field math will +// show. // // Why log_blowup=1 (vs miden-vm production's 3): // - Smaller LDE → faster prove → CI-friendly. -// - This is a perf-tracking bench, not a security parameter; the -// proven-soundness number from this config is irrelevant. We just -// need the same arithmetic shape on every run. +// - This is a perf-tracking bench, not a security parameter; the proven-soundness number from +// this config is irrelevant. We just need the same arithmetic shape on every run. // // Trace size is parameterised by `log_n` so we can tune CI runtime // without rebuilding the wasm. Default callers should pass log_n=12 @@ -259,7 +251,7 @@ bench_packed_permute!(bench_poseidon2_packed_permute, Poseidon2Permutation256); pub fn bench_lifted_stark_prove_blake3(num_runs: u32, log_n: u32) -> Vec { let n = 1usize << log_n; let mut rng = ChaCha20Rng::seed_from_u64(SEED); - let inputs: alloc::vec::Vec<[u32; 24]> = (0..n) + let inputs: Vec<[u32; 24]> = (0..n) .map(|_| { let mut row = [0u32; 24]; for v in &mut row { @@ -278,12 +270,8 @@ pub fn bench_lifted_stark_prove_blake3(num_runs: u32, log_n: u32) -> Vec { // with no PoW, this minimises wall-clock per prove while still // exercising the full pipeline. let pcs = PcsParams::new( - /* log_blowup */ 1, - /* log_folding_arity */ 2, - /* log_final_degree */ 7, - /* folding_pow_bits */ 0, - /* deep_pow_bits */ 0, - /* num_queries */ 27, + /* log_blowup */ 1, /* log_folding_arity */ 2, /* log_final_degree */ 7, + /* folding_pow_bits */ 0, /* deep_pow_bits */ 0, /* num_queries */ 27, /* query_pow_bits */ 0, ) .expect("invalid PCS params"); @@ -298,7 +286,7 @@ pub fn bench_lifted_stark_prove_blake3(num_runs: u32, log_n: u32) -> Vec { let aux = ZeroAuxBuilder::dummy(); let perf = web_sys::window().expect("no window").performance().expect("no performance"); - let mut samples = alloc::vec::Vec::with_capacity(num_runs as usize); + let mut samples = Vec::with_capacity(num_runs as usize); for _ in 0..num_runs { // Fresh witness + challenger per run — prove_multi consumes the // challenger, and we want byte-identical inputs across runs. From 120c50b81c4a02469f8793b8373e2c372396aa83 Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Sun, 10 May 2026 13:25:27 +0200 Subject: [PATCH 05/10] fix(bench): driver hung after writing JSON; explicit process.exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wasm bench job on GHA timed out at the 15-min cap even though every bench finished and produced a median in ~2 min. Cause: after `main()` returns, Node's event loop stayed alive (open file descriptors from the `npx serve` child's piped stdio + Playwright's lingering handles), so the script never exited until GHA killed it. Fix: explicit `process.exit(0)` after main() resolves. SIGKILL on the serve child alone doesn't reliably clear the parent's piped fds; just exit when work is done. Also bumped the wasm job's `timeout-minutes` 15 → 20 to give comfortable headroom: GHA ubuntu-latest is ~2.7× slower than Apple M5 on these benches (measured), bringing total step time to ~6-7 min in the typical case. Verified locally: driver now exits cleanly in 42s (was hanging indefinitely). All 14 medians produced as before. --- .github/workflows/bench.yml | 7 ++++++- miden-bench-wasm/driver.mjs | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 34da9d146f..4481dc49b0 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -50,7 +50,12 @@ jobs: bench-wasm: name: WASM perf (Chromium / V8) runs-on: ubuntu-latest - timeout-minutes: 15 + # Local Apple M5 wall-clock for the full bench is ~40s; GHA ubuntu- + # latest x86_64 is ~2.7× slower across these benches (measured), + # putting the bench step at ~2 min. Plus wasm-pack build (~3 min) + # + Chromium install (~30s) + setup overhead = ~6-7 min typical. + # 20 min cap leaves comfortable headroom for the worst-case run. + timeout-minutes: 20 steps: - uses: actions/checkout@v4 diff --git a/miden-bench-wasm/driver.mjs b/miden-bench-wasm/driver.mjs index 5769e73434..9ced817e5a 100644 --- a/miden-bench-wasm/driver.mjs +++ b/miden-bench-wasm/driver.mjs @@ -174,7 +174,15 @@ async function main() { process.stdout.write(JSON.stringify(results, null, 2) + "\n"); } -main().catch((err) => { - process.stderr.write(`driver failed: ${err.stack || err.message}\n`); - process.exit(1); -}); +// Explicit `process.exit(0)` after main() returns. Without this, Node's +// event loop stays alive (open file descriptors from the `npx serve` +// child's piped stdio + Playwright's lingering handles), and the driver +// hangs until GHA's job timeout fires — even though every bench has +// completed and the JSON has been written. SIGKILL on the child doesn't +// reliably clear the parent's piped fds. Just exit. +main() + .then(() => process.exit(0)) + .catch((err) => { + process.stderr.write(`driver failed: ${err.stack || err.message}\n`); + process.exit(1); + }); From 214b0ac67d674b7a05bed1118a8edd3a8c8fbc44 Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Thu, 14 May 2026 17:52:40 +0200 Subject: [PATCH 06/10] =?UTF-8?q?ci(bench):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20split=20privileges,=20pin=20actions,=20fix=20no-std=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review-flagged issues: 1. Privilege escalation. The bench jobs run PR-controlled Rust / npm / Node while holding contents:write (for gh-pages push) and pull-requests:write (for sticky comments). actions/checkout persists the GHA token in .git/config by default, giving PR-controlled build scripts (build.rs, npm postinstall) access to it before the benchmark step runs. Split into two layers, mirroring the miden-vm non-regression workflows: bench-{wasm,native} run with contents:read and persist-credentials:false, upload a results artifact, and never see a write token; publish-{wasm,native} check out the default branch (trusted ref), download the artifact, and run benchmark-action/github-action-benchmark with the privileged token. The PR's tree never reaches a step that has write scope. 2. Unpinned actions. Floating @v4 / @v0.4.0 / @v1 tags can be moved by the action author. Pinned to commit SHAs with readable pin@ comments, matching the rest of the repo's workflows: - actions/checkout @34e114876b... pin@v4 - actions/upload-artifact @ea165f8d65... pin@v4 - actions/download-artifact @d3f86a106a... pin@v4 - jetli/wasm-pack-action @0d096b08b4... pin@v0.4.0 - benchmark-action/github-action-benchmark @52576c92bc... pin@v1.22.1 3. No-std gate corruption. miden-bench-wasm hard-depends on miden-crypto/std (browser benches need std for the once_cell real- sync path). `make build-no-std` runs `cargo build --no-default-features --target wasm32-unknown-unknown` without --workspace; with miden-bench-wasm as a default workspace member, Cargo feature unification silently enables std on miden-crypto and the no-std check passes without actually validating the no-std config. Added `default-members` to the workspace manifest, excluding miden-bench-wasm. `cargo bench` / `wasm-pack` invoke the bench crate by explicit package path, so the bench workflow is unaffected. Verified locally: build-no-std now finishes without compiling miden-bench-wasm. --- .github/workflows/bench.yml | 158 +++++++++++++++++++++++++----------- Cargo.toml | 22 +++++ 2 files changed, 131 insertions(+), 49 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 4481dc49b0..6134d3850c 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -11,23 +11,27 @@ # runner. Tracks the same primitives at native speed for # comparison + regression detection. # -# Both jobs feed `benchmark-action/github-action-benchmark`, which: +# Each bench job is paired with a trusted `publish-*` job that downloads +# its artifact and calls `benchmark-action/github-action-benchmark` with +# the write-capable token. The split is deliberate: the bench jobs run +# PR-controlled code (Rust, npm install, Node) and therefore MUST NOT +# hold credentials, so they run with `contents: read` and +# `persist-credentials: false`. The publish jobs check out the default +# branch (trusted ref) before running the action, so the PR's tree never +# reaches a step that has write scope. This is the same split miden-vm's +# non-regression workflows use. +# +# `benchmark-action/github-action-benchmark`: # - Stores each metric over time on the `gh-pages` branch under # `bench/` (separate from the existing `docs/` subdir used by docs.yml, # so the two don't collide). # - Posts a sticky PR comment showing the diff vs the latest `next` # baseline. Header per job, so wasm and native each get their own # comment that updates in place. -# - Alerts (writes a comment header) on regression > 15 %. Doesn't fail +# - Alerts (writes a comment header) on regression > 10 %. Doesn't fail # the workflow — GHA Linux runners share CPUs and the noise floor on # hash benches is real. False-positive alerts are cheap to ignore; # false-negative reverts are expensive. -# -# This is deliberately one workflow with two jobs (rather than two -# separate workflow files): both produce sticky comments to the same PR, -# both use the same gh-pages branch, both share the same triage -# discipline. Putting them next to each other in one file keeps the -# regression-tracking story coherent. name: bench @@ -37,9 +41,11 @@ on: branches: [next] workflow_dispatch: +# Default to read-only at workflow scope. Privileged scopes are granted +# per-job and only on the trusted publish jobs that do NOT execute any +# PR-controlled code. permissions: - contents: write # github-action-benchmark needs to push to gh-pages - pull-requests: write # for sticky PR comment + contents: read # Skip duplicate runs on PR sync events. concurrency: @@ -56,8 +62,16 @@ jobs: # + Chromium install (~30s) + setup overhead = ~6-7 min typical. # 20 min cap leaves comfortable headroom for the worst-case run. timeout-minutes: 20 + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 + with: + # The default checkout writes the GHA token to `.git/config`, + # which PR-controlled build scripts (build.rs, npm postinstall, + # etc.) would then have access to. Strip it — this job has no + # business holding any credential. + persist-credentials: false - name: Install Rust + wasm32 target run: | @@ -65,9 +79,8 @@ jobs: rustup target add wasm32-unknown-unknown - name: Install wasm-pack - # Pinned major (`v0.13`) — wasm-pack pre-compiled binaries break - # at minor bumps occasionally. Bump deliberately, never floating. - uses: jetli/wasm-pack-action@v0.4.0 + # Pinned to v0.4.0 SHA. The action installs wasm-pack v0.13.1. + uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa # pin@v0.4.0 with: version: "v0.13.1" @@ -108,48 +121,26 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Upload raw results as artifact - # Keeps the full per-batch sample distribution + driver stderr - # for post-hoc analysis when an alert fires. The - # github-action-benchmark store only retains the median. - uses: actions/upload-artifact@v4 + # Picked up by the trusted `publish-wasm` job, which runs the + # benchmark-action against this file with write-scope token. Also + # keeps the full per-batch sample distribution for post-hoc + # analysis when an alert fires. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # pin@v4 with: name: bench-wasm-results path: miden-bench-wasm/results.json retention-days: 30 - - name: Track + alert on regression - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: customSmallerIsBetter - output-file-path: miden-bench-wasm/results.json - # Separate gh-pages subdir so this never collides with docs.yml's - # `destination_dir: docs` deploy (which lives at /docs/). - benchmark-data-dir-path: bench/wasm - gh-pages-branch: gh-pages - # Push baseline updates ONLY on push to next (not on PR runs — - # those compare against the existing baseline, they don't update it). - auto-push: ${{ github.event_name == 'push' }} - comment-on-alert: true - # Regression alert at 10 %. Run-to-run noise on GHA shared - # runners is real but accepting 15-20 % headroom would defeat - # the purpose: a 15 % regression IS a regression worth - # investigating, not the noise floor. Drive variance down - # (longer batches, more samples) to fit under this threshold - # rather than widening the threshold to fit measured noise. - # See README "Noise reduction" for the roadmap to tighten further. - alert-threshold: "110%" - fail-on-alert: false # warn via comment, don't fail the PR - summary-always: true # keep the diff comment fresh on each run - comment-always: ${{ github.event_name == 'pull_request' }} - alert-comment-cc-users: '@WiktorStarczewski' - github-token: ${{ secrets.GITHUB_TOKEN }} - bench-native: name: Native perf (Linux x86_64) runs-on: ubuntu-latest timeout-minutes: 20 + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 + with: + persist-credentials: false - name: Install Rust run: rustup update --no-self-update @@ -177,17 +168,86 @@ jobs: | tee bench-native.txt - name: Upload raw results as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # pin@v4 with: name: bench-native-results path: bench-native.txt retention-days: 30 + publish-wasm: + name: Publish WASM bench + needs: bench-wasm + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # github-action-benchmark pushes to gh-pages on `push to next` + pull-requests: write # for sticky PR comment on pull_request runs + steps: + # Explicitly check out the default branch (trusted ref). On a + # `pull_request` event, the default `actions/checkout` resolves to + # the PR's merge ref — that's PR-controlled code, which has no + # business being on disk in a job that holds the write token. + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Download bench results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 + with: + name: bench-wasm-results + path: results + + - name: Track + alert on regression + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # pin@v1.22.1 + with: + tool: customSmallerIsBetter + output-file-path: results/results.json + # Separate gh-pages subdir so this never collides with docs.yml's + # `destination_dir: docs` deploy (which lives at /docs/). + benchmark-data-dir-path: bench/wasm + gh-pages-branch: gh-pages + # Push baseline updates ONLY on push to next (not on PR runs — + # those compare against the existing baseline, they don't update it). + auto-push: ${{ github.event_name == 'push' }} + comment-on-alert: true + # Regression alert at 10 %. Run-to-run noise on GHA shared + # runners is real but accepting 15-20 % headroom would defeat + # the purpose: a 15 % regression IS a regression worth + # investigating, not the noise floor. Drive variance down + # (longer batches, more samples) to fit under this threshold + # rather than widening the threshold to fit measured noise. + # See README "Noise reduction" for the roadmap to tighten further. + alert-threshold: "110%" + fail-on-alert: false # warn via comment, don't fail the PR + summary-always: true # keep the diff comment fresh on each run + comment-always: ${{ github.event_name == 'pull_request' }} + alert-comment-cc-users: '@WiktorStarczewski' + github-token: ${{ secrets.GITHUB_TOKEN }} + + publish-native: + name: Publish native bench + needs: bench-native + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Download bench results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 + with: + name: bench-native-results + path: results + - name: Track + alert on regression - uses: benchmark-action/github-action-benchmark@v1 + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # pin@v1.22.1 with: tool: cargo - output-file-path: bench-native.txt + output-file-path: results/bench-native.txt benchmark-data-dir-path: bench/native gh-pages-branch: gh-pages auto-push: ${{ github.event_name == 'push' }} diff --git a/Cargo.toml b/Cargo.toml index 0263a4d184..e4cd5e7463 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,28 @@ members = [ "stark/miden-stark-transcript", "stark/miden-stateful-hasher", ] +# `miden-bench-wasm` is intentionally NOT a default member. It hard- +# depends on `miden-crypto/std` (browser benches need `std` for the +# `once_cell` real-sync path; the `critical-section` fallback requires +# a host-provided impl that browsers don't supply), so including it in +# the default workspace build would cause `make build-no-std` — +# `cargo build --no-default-features --target wasm32-unknown-unknown`, +# no `--workspace` flag — to silently enable `std` on `miden-crypto` +# through Cargo feature unification. The no-std gate would pass while +# not actually validating `miden-crypto` in the intended no-std config. +# `cargo bench`/`wasm-pack` invoke the bench crate via explicit package +# paths, so this exclusion has no effect on the bench workflow itself. +default-members = [ + "miden-bench", + "miden-crypto", + "miden-crypto-derive", + "miden-field", + "miden-serde-utils", + "stark/miden-lifted-air", + "stark/miden-lifted-stark", + "stark/miden-stark-transcript", + "stark/miden-stateful-hasher", +] resolver = "3" [workspace.package] From ae75f95cdb7618f5ec01905696459fbc53333006 Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Thu, 14 May 2026 18:19:30 +0200 Subject: [PATCH 07/10] ci(bench): satisfy actionlint shellcheck + zizmor artipacked audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - actionlint: shellcheck SC2002 'useless cat'. Replaced `cat results.json | wc -l` with `wc -l < results.json`. - zizmor: `artipacked` warned that the publish-* jobs' checkouts left the token persisted in .git/config. The benchmark action reads its token from the `github-token` input rather than the persisted remote credential, so persist-credentials: false is safe here and closes the warning. doc and doc-build are pre-existing failures on next (broken intra-doc links in merkle/smt/large_forest/backend/persistent/snapshot.rs from PR #989) — unrelated, not addressed here. --- .github/workflows/bench.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6134d3850c..8357f1a278 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -109,7 +109,7 @@ jobs: working-directory: miden-bench-wasm run: | node driver.mjs > results.json - echo "results=$(cat results.json | wc -l) entries written" + echo "results=$(wc -l < results.json) entries written" # Surface the per-bench medians in the workflow summary so a # human can eyeball them without clicking into the artifact. { @@ -190,6 +190,11 @@ jobs: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 with: ref: ${{ github.event.repository.default_branch }} + # `benchmark-action/github-action-benchmark` reads its credential + # from the `github-token` input rather than `.git/config`, so the + # persisted token isn't needed. Strip it to satisfy zizmor's + # `artipacked` audit. + persist-credentials: false - name: Download bench results uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 @@ -236,6 +241,11 @@ jobs: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 with: ref: ${{ github.event.repository.default_branch }} + # `benchmark-action/github-action-benchmark` reads its credential + # from the `github-token` input rather than `.git/config`, so the + # persisted token isn't needed. Strip it to satisfy zizmor's + # `artipacked` audit. + persist-credentials: false - name: Download bench results uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 From 4718b59e87d029ac7574a718331931ba6e0c76e6 Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Thu, 14 May 2026 19:46:09 +0200 Subject: [PATCH 08/10] ci(bench): fork-PR gate, pipefail, serialize publishers (2nd review pass) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups from @huitseeker's second review pass: 1. Fork PRs no longer crash the publish step. GitHub silently strips `pull-requests: write` from `GITHUB_TOKEN` for fork PRs even when the workflow declares it, so `comment-always: true` would fail with 'Resource not accessible by integration'. Gated to same-repo PRs (and push events) — fork PRs still get the artifact + workflow summary, just no sticky comment. 2. `set -euo pipefail` on both bench shell blocks. Without it, `cargo bench ... | tee bench-native.txt` silently turns a compile or runtime failure into 'exit 0, bad file uploaded' because tee's exit code is the one observed. Same hazard on the wasm side where `node driver.mjs > results.json` leaves a partial file behind on a panicking driver. Both blocks now fail fast. 3. Publish jobs serialized on gh-pages. On a push-to-next event both publishers fetch gh-pages, append a commit, and push concurrently; the loser hits non-fast-forward and the metric is lost. Added `needs: publish-wasm` to `publish-native` so they run in series. Cost: ~20 s on push events only (PR runs don't push). Keeps per-job permission scope narrow vs the alternative of merging the publishers into one job. --- .github/workflows/bench.yml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8357f1a278..52301ad29c 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -108,6 +108,12 @@ jobs: id: run working-directory: miden-bench-wasm run: | + # `pipefail` is load-bearing: the driver streams to a file via + # a shell redirect, so a panicking driver would still leave a + # partial `results.json` on disk; `set -e` ensures the step + # fails fast and the trusted publish job never sees a bad + # baseline. + set -euo pipefail node driver.mjs > results.json echo "results=$(wc -l < results.json) entries written" # Surface the per-bench medians in the workflow summary so a @@ -163,6 +169,12 @@ jobs: # - encryption / dsa: defer pending review of which sub-benches # are stable. run: | + # `pipefail` ensures a failed `cargo bench` (compile or run) + # propagates through the `tee` and fails the step. Without + # it, only tee's exit status is observed and a broken bench + # would silently produce an empty/partial `bench-native.txt` + # for the trusted publish job to ingest as a baseline. + set -euo pipefail cargo bench --bench hash --bench word --bench transpose \ -p miden-crypto -- --output-format bencher 2>&1 \ | tee bench-native.txt @@ -225,13 +237,28 @@ jobs: alert-threshold: "110%" fail-on-alert: false # warn via comment, don't fail the PR summary-always: true # keep the diff comment fresh on each run - comment-always: ${{ github.event_name == 'pull_request' }} + # Gate the sticky comment to same-repo PRs. GitHub silently + # strips `pull-requests: write` from `GITHUB_TOKEN` for fork + # PRs regardless of the workflow-declared permissions, so + # `comment-always: true` would otherwise crash the publish + # step with "Resource not accessible by integration" on every + # external contribution. Fork PRs still get the workflow + # summary table and the artifact — just no PR comment. + comment-always: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} alert-comment-cc-users: '@WiktorStarczewski' github-token: ${{ secrets.GITHUB_TOKEN }} publish-native: name: Publish native bench - needs: bench-native + # Serialized after `publish-wasm` so the two never race on gh-pages. + # Both jobs `git fetch gh-pages → commit → push`, and on a `push to + # next` event both fire concurrently — whichever pushes second hits + # non-fast-forward and the metric for that job is lost. Running them + # in series costs ~20 s on push events and zero on PR runs (which + # don't push). Cheaper than the alternative (merging the publishers + # into one job) because it keeps the per-job permission scope + # narrow and lets each artifact pass/fail independently. + needs: [bench-native, publish-wasm] runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -271,6 +298,7 @@ jobs: alert-threshold: "110%" fail-on-alert: false summary-always: true - comment-always: ${{ github.event_name == 'pull_request' }} + # See `publish-wasm` for why this is gated to same-repo PRs. + comment-always: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} alert-comment-cc-users: '@WiktorStarczewski' github-token: ${{ secrets.GITHUB_TOKEN }} From da13e80ad1afbfe0ce3d8d6734da1d5ea5af0628 Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Thu, 14 May 2026 19:49:34 +0200 Subject: [PATCH 09/10] ci(bench): pull_request_target for fork PRs (matches web-sdk pattern) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the fork-PR gate from the previous commit with the right fix: keep posting the sticky comment on fork PRs by running under `pull_request_target` instead of `pull_request`. Under `pull_request`, GitHub silently strips `pull-requests: write` and `contents: write` from `GITHUB_TOKEN` for fork PRs regardless of declared `permissions:`. The previous commit worked around this by skipping the comment on fork PRs entirely — which is the wrong answer: fork contributors deserve the same perf-tracking signal as same-repo PRs. `pull_request_target` keeps the token alive on fork PRs because the workflow runs in the base repo's context. SAFETY note (and why `pull_request_target` is OK despite the 'pwn-request' risk): the bench jobs (the only steps that execute PR-supplied code) declare per-job `permissions: contents: read`, check out the PR head with `persist-credentials: false`, and reference no `secrets.*`. So even with the PR's tree on disk, no privileged token is reachable from build.rs / npm postinstall / Node. The publish jobs check out the BASE ref (which is `pull_request_target`'s default), so PR-supplied code never touches a step with the write token. The workflow file itself comes from base under `pull_request_target`, so a malicious PR can't change the workflow logic against itself. Same pattern is used in web-sdk's check-linked-client-pr.yml for the same reason. --- .github/workflows/bench.yml | 61 +++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 52301ad29c..d724dba1aa 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -35,8 +35,30 @@ name: bench +# `pull_request_target` (not `pull_request`) so the publish job's +# `GITHUB_TOKEN` keeps `pull-requests: write` and `contents: write` on +# fork PRs. Under `pull_request`, GitHub silently strips both scopes +# from `GITHUB_TOKEN` for fork PRs regardless of what `permissions:` +# declares, which makes the sticky comment step crash with "Resource +# not accessible by integration" on every external contribution. The +# same pattern is used in web-sdk's `check-linked-client-pr.yml` for +# the same reason. +# +# SAFETY: `pull_request_target` is safe here because the bench jobs +# (the only steps that execute PR-controlled code) hold no privileged +# token. Each bench job declares per-job `permissions: contents: read`, +# checks out the PR head with `persist-credentials: false`, and +# doesn't reference any `secrets.*` — so even though the PR's tree is +# on disk, build.rs / npm postinstall / Node has nothing privileged to +# reach for. The trusted publish jobs check out the base ref (which is +# what `pull_request_target` defaults to anyway), so PR-supplied code +# never touches a step that has the write token. Under +# `pull_request_target` the workflow file itself is the base's, not +# the PR's, so a malicious PR can't edit the workflow logic that's run +# against them. on: - pull_request: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] push: branches: [next] workflow_dispatch: @@ -67,6 +89,11 @@ jobs: steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 with: + # Under `pull_request_target`, the default checkout is the + # BASE branch — but we need to benchmark the PR's code, so + # explicitly target the PR head SHA. `github.sha` is the + # fallback for push and workflow_dispatch events. + ref: ${{ github.event.pull_request.head.sha || github.sha }} # The default checkout writes the GHA token to `.git/config`, # which PR-controlled build scripts (build.rs, npm postinstall, # etc.) would then have access to. Strip it — this job has no @@ -146,6 +173,9 @@ jobs: steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 with: + # See `bench-wasm` for why both `ref:` and `persist-credentials` + # are set explicitly under `pull_request_target`. + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Install Rust @@ -193,12 +223,12 @@ jobs: timeout-minutes: 10 permissions: contents: write # github-action-benchmark pushes to gh-pages on `push to next` - pull-requests: write # for sticky PR comment on pull_request runs + pull-requests: write # for sticky PR comment on pull_request_target runs steps: - # Explicitly check out the default branch (trusted ref). On a - # `pull_request` event, the default `actions/checkout` resolves to - # the PR's merge ref — that's PR-controlled code, which has no - # business being on disk in a job that holds the write token. + # Explicitly check out the default branch (trusted ref). Under + # `pull_request_target` this would be the default anyway, but + # spelling it out keeps the invariant ("publish jobs never touch + # PR-supplied code") visible to anyone reading the workflow. - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 with: ref: ${{ github.event.repository.default_branch }} @@ -237,14 +267,12 @@ jobs: alert-threshold: "110%" fail-on-alert: false # warn via comment, don't fail the PR summary-always: true # keep the diff comment fresh on each run - # Gate the sticky comment to same-repo PRs. GitHub silently - # strips `pull-requests: write` from `GITHUB_TOKEN` for fork - # PRs regardless of the workflow-declared permissions, so - # `comment-always: true` would otherwise crash the publish - # step with "Resource not accessible by integration" on every - # external contribution. Fork PRs still get the workflow - # summary table and the artifact — just no PR comment. - comment-always: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + # `pull_request_target` (vs. `pull_request`) is what keeps the + # write token alive for fork PRs — see the header comment. + # The event name we match here is `pull_request_target` + # because that's what `github.event_name` resolves to under + # this trigger; the value is NOT `pull_request`. + comment-always: ${{ github.event_name == 'pull_request_target' }} alert-comment-cc-users: '@WiktorStarczewski' github-token: ${{ secrets.GITHUB_TOKEN }} @@ -298,7 +326,8 @@ jobs: alert-threshold: "110%" fail-on-alert: false summary-always: true - # See `publish-wasm` for why this is gated to same-repo PRs. - comment-always: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + # See `publish-wasm` and the header comment for why we match + # on `pull_request_target` rather than `pull_request`. + comment-always: ${{ github.event_name == 'pull_request_target' }} alert-comment-cc-users: '@WiktorStarczewski' github-token: ${{ secrets.GITHUB_TOKEN }} From 08ad7c6c5980f1e73d75600e00af0f49dc39545b Mon Sep 17 00:00:00 2001 From: Wiktor Starczewski Date: Thu, 14 May 2026 19:59:24 +0200 Subject: [PATCH 10/10] =?UTF-8?q?ci(bench):=20workflow=5Frun=20split=20?= =?UTF-8?q?=E2=80=94=20close=20GHAS=20cache-poisoning=20alerts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GHAS's `actions/cache-poisoning/poisonable-step` rule fired on the previous `pull_request_target` version of bench.yml: any step under `pull_request_target` that executes untrusted code holds an implicit `ACTIONS_RUNTIME_TOKEN` with Actions-cache write, regardless of declared `permissions:`. The fix is the workflow_run split that GitHub Security Lab documents (https://securitylab.github.com/research/github-actions-preventing-pwn-requests/): untrusted bench in one workflow with no privileged token, trusted publish in a second workflow that fires via workflow_run. bench.yml (UNTRUSTED): - back to `pull_request` trigger (no privilege downgrade concern because we no longer try to comment from here) - workflow- and job-scope `permissions: contents: read` - bench-wasm + bench-native upload results + a metadata.json containing the PR number, head SHA, and event name (fork PRs aren't reachable via workflow_run.pull_requests, so we have to stash the PR identity ourselves) - no publish jobs at all bench-publish.yml (TRUSTED, new): - triggered by `workflow_run: workflows: [bench]: types: [completed]` - runs in the BASE repo's context, full write token even for fork PRs - doesn't execute PR code: only consumes the bench-results JSON artifact via cross-workflow download with run-id - validates the PR-author-controlled metadata.json against tight formats (pr_number ^[0-9]+$, pr_head_sha ^[0-9a-f]{40}$, event whitelist) before trusting any value from it - benchmark-action with comment-always disabled (it can't find the PR under a workflow_run event anyway); we post our own sticky comment via actions/github-script reading the local benchmark-data-repository clone for the diff against baseline - publish-native serialized after publish-wasm to keep gh-pages pushes ordered Side effects: - The first PR after this lands will have no publish coverage (the workflow_run trigger reads bench-publish.yml from `next`, which this PR puts there). After merge, future PRs are fully covered. - Fork PRs now get a sticky comment again — same as before the GHAS-driven revert, but via a path the cache-poisoning rule accepts. --- .github/workflows/bench-publish.yml | 373 ++++++++++++++++++++++++++++ .github/workflows/bench.yml | 240 +++++------------- 2 files changed, 434 insertions(+), 179 deletions(-) create mode 100644 .github/workflows/bench-publish.yml diff --git a/.github/workflows/bench-publish.yml b/.github/workflows/bench-publish.yml new file mode 100644 index 0000000000..9b39445931 --- /dev/null +++ b/.github/workflows/bench-publish.yml @@ -0,0 +1,373 @@ +# Performance regression tracking for miden-crypto — TRUSTED stage. +# +# Triggered when `bench.yml` finishes. Runs in the base repo's context +# with the BASE repo's `GITHUB_TOKEN`, so it has write scope even for +# PRs from forks (the `pull_request` event would silently strip those +# scopes for fork PRs, breaking the comment step). This workflow does +# NOT execute any PR-supplied code — the only thing crossing the trust +# boundary is the bench-results JSON artifact uploaded by `bench.yml`. +# Cross-workflow artifact download is via `actions/download-artifact` +# with `run-id: ${{ github.event.workflow_run.id }}`, which fetches +# whatever bytes the artifact happens to contain without ever materialising +# the PR's tree on disk in this job. +# +# Two jobs: +# - publish-wasm: pushes the WASM perf metric onto gh-pages (push to +# next only) and posts the sticky PR comment. +# - publish-native: same for the native metric, serialized after +# publish-wasm so the two never race on gh-pages. +# +# Why a separate workflow instead of `pull_request_target`: GHAS's +# `actions/cache-poisoning/poisonable-step` rule flags any step under +# `pull_request_target` that executes untrusted code, because the +# implicit `ACTIONS_RUNTIME_TOKEN` grants Actions-cache write to the +# job regardless of `permissions:`. The `workflow_run` split keeps the +# untrusted bench execution in a job with no privileged token at all, +# and brings privilege online only in this workflow — which doesn't +# touch PR code. Documented pattern from GitHub Security Lab. + +name: bench-publish + +on: + workflow_run: + workflows: [bench] + types: [completed] + +permissions: + contents: write # github-action-benchmark pushes to gh-pages on push to next + pull-requests: write # for the sticky comment + actions: read # to download artifacts from the triggering workflow run + +# Serialize publishes per source branch so two pushes to next don't race +# on gh-pages. `cancel-in-progress: false` because cancelling mid-push +# could leave gh-pages with a half-applied change. +concurrency: + group: bench-publish-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + publish-wasm: + name: Publish WASM bench + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # Under `workflow_run`, the default ref is the workflow file's + # branch on the BASE repo (so for a PR from a fork, this is the + # base's `next`, not the fork's tree). Explicit `persist-credentials: + # false` because benchmark-action gets its token via the + # `github-token` input, not via `.git/config` — no need to hand + # the persisted token to any subsequent step. + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 + with: + persist-credentials: false + + - name: Download bench-wasm artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 + with: + name: bench-wasm-results + path: artifacts/wasm + # Cross-workflow download: target the triggering workflow's + # run-id and provide a token. Without these, the action + # defaults to looking in the current workflow's artifacts + # (and finds nothing). + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read PR metadata + id: meta + run: | + set -euo pipefail + meta="artifacts/wasm/metadata.json" + if [ ! -f "$meta" ]; then + echo "::error::metadata.json missing from bench-wasm-results artifact" + exit 1 + fi + # PR-author-controlled file — validate before trusting. + event=$(jq -r '.event // ""' "$meta") + pr_number=$(jq -r '.pr_number // ""' "$meta") + pr_head_sha=$(jq -r '.pr_head_sha // ""' "$meta") + + # pr_number must be a positive integer or empty. + if [ -n "$pr_number" ] && ! [[ "$pr_number" =~ ^[0-9]+$ ]]; then + echo "::error::pr_number from metadata is not a positive integer: '$pr_number'" + exit 1 + fi + # pr_head_sha must be a 40-char hex string or empty. + if [ -n "$pr_head_sha" ] && ! [[ "$pr_head_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::pr_head_sha from metadata is not a 40-char SHA: '$pr_head_sha'" + exit 1 + fi + # event must be one of the known triggers. + case "$event" in + pull_request|push|workflow_dispatch) ;; + *) + echo "::error::unrecognised event in metadata: '$event'" + exit 1 + ;; + esac + + { + echo "event=$event" + echo "pr_number=$pr_number" + echo "pr_head_sha=$pr_head_sha" + } >> "$GITHUB_OUTPUT" + + - name: Track + push baseline + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # pin@v1.22.1 + with: + tool: customSmallerIsBetter + output-file-path: artifacts/wasm/results.json + # Separate gh-pages subdir so this never collides with docs.yml's + # `destination_dir: docs` deploy (which lives at /docs/). + benchmark-data-dir-path: bench/wasm + gh-pages-branch: gh-pages + # Push baseline updates ONLY on push to next (PR runs compare + # against the existing baseline, they don't update it). + auto-push: ${{ steps.meta.outputs.event == 'push' }} + alert-threshold: "110%" + fail-on-alert: false + summary-always: true + # benchmark-action's comment-posting relies on the + # `context.issue.number` that GitHub populates for + # `pull_request` events. Under `workflow_run` the event + # context is `workflow_run` and `issue.number` is undefined — + # so the action's comment step is a no-op here. We post our + # own sticky comment below using the PR number from the + # validated metadata file. + comment-always: false + comment-on-alert: false + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Post sticky PR comment + if: ${{ steps.meta.outputs.event == 'pull_request' && steps.meta.outputs.pr_number != '' }} + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # pin@v7 + env: + PR_NUMBER: ${{ steps.meta.outputs.pr_number }} + PR_HEAD_SHA: ${{ steps.meta.outputs.pr_head_sha }} + BENCH_NAME: WASM perf (Chromium / V8) + MARKER: "" + DATA_PATH: ./benchmark-data-repository/bench/wasm/data.js + with: + script: | + const fs = require('fs'); + + // Parse the local gh-pages clone benchmark-action just + // populated. The file is a JS assignment, not pure JSON — + // strip the `window.BENCHMARK_DATA = ` prefix and the + // trailing semicolon. + const raw = fs.readFileSync(process.env.DATA_PATH, 'utf8'); + const match = raw.match(/window\.BENCHMARK_DATA\s*=\s*(\{[\s\S]+\});?\s*$/); + if (!match) { + core.setFailed(`could not parse ${process.env.DATA_PATH}`); + return; + } + const data = JSON.parse(match[1]); + const history = data.entries[process.env.BENCH_NAME]; + if (!history || history.length === 0) { + core.warning(`no history yet for "${process.env.BENCH_NAME}"; skipping comment`); + return; + } + + const current = history[history.length - 1]; + const baseline = history.length > 1 ? history[history.length - 2] : null; + + const fmt = (v, unit) => `${Number(v).toLocaleString('en-US', { maximumFractionDigits: 0 })} ${unit}`; + const pctDelta = (cur, prev) => { + if (prev === undefined || prev === null || prev === 0) return 'n/a'; + const pct = ((cur - prev) / prev) * 100; + const sign = pct >= 0 ? '+' : ''; + const tag = pct > 10 ? ' ⚠️' : pct < -10 ? ' ⚡' : ''; + return `${sign}${pct.toFixed(1)}%${tag}`; + }; + + let body = `${process.env.MARKER}\n## ${process.env.BENCH_NAME}\n\n`; + body += '| Bench | Baseline | Current | Δ |\n|---|---|---|---|\n'; + for (const bench of current.benches) { + const prev = baseline?.benches.find((b) => b.name === bench.name); + body += `| \`${bench.name}\` | ${prev ? fmt(prev.value, prev.unit) : '—'} | ${fmt(bench.value, bench.unit)} | ${pctDelta(bench.value, prev?.value)} |\n`; + } + body += `\nBaseline: latest \`${data.repoUrl?.split('/').slice(-2).join('/') || 'gh-pages'}\` entry. Comparing PR head \`${(process.env.PR_HEAD_SHA || '').slice(0, 8)}\`. _cc @WiktorStarczewski on regressions > 10%._\n`; + + const prNumber = parseInt(process.env.PR_NUMBER, 10); + + // Sticky-comment: find existing by marker, update; else create. + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + }); + const existing = comments.find((c) => c.body && c.body.startsWith(process.env.MARKER)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); + } + + publish-native: + name: Publish native bench + # Serialized after publish-wasm so the two never race on gh-pages. + # Both jobs `git fetch gh-pages → commit → push`, and on a push-to- + # next event they fire concurrently — whichever pushes second hits + # non-fast-forward and the metric for that job is lost. Running them + # in series costs ~20 s on push events and zero on PR runs (which + # don't push). Cheaper than merging the publishers into one job + # because the per-job permission scope stays narrow and each + # artifact passes/fails independently. + needs: publish-wasm + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 + with: + persist-credentials: false + + - name: Download bench-native artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 + with: + name: bench-native-results + path: artifacts/native + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read PR metadata + id: meta + # See `publish-wasm.Read PR metadata` for why each field is + # validated against a tight format before being trusted. + run: | + set -euo pipefail + meta="artifacts/native/metadata.json" + if [ ! -f "$meta" ]; then + echo "::error::metadata.json missing from bench-native-results artifact" + exit 1 + fi + event=$(jq -r '.event // ""' "$meta") + pr_number=$(jq -r '.pr_number // ""' "$meta") + pr_head_sha=$(jq -r '.pr_head_sha // ""' "$meta") + if [ -n "$pr_number" ] && ! [[ "$pr_number" =~ ^[0-9]+$ ]]; then + echo "::error::pr_number from metadata is not a positive integer: '$pr_number'" + exit 1 + fi + if [ -n "$pr_head_sha" ] && ! [[ "$pr_head_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::pr_head_sha from metadata is not a 40-char SHA: '$pr_head_sha'" + exit 1 + fi + case "$event" in + pull_request|push|workflow_dispatch) ;; + *) + echo "::error::unrecognised event in metadata: '$event'" + exit 1 + ;; + esac + { + echo "event=$event" + echo "pr_number=$pr_number" + echo "pr_head_sha=$pr_head_sha" + } >> "$GITHUB_OUTPUT" + + - name: Track + push baseline + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # pin@v1.22.1 + with: + tool: cargo + output-file-path: artifacts/native/bench-native.txt + benchmark-data-dir-path: bench/native + gh-pages-branch: gh-pages + auto-push: ${{ steps.meta.outputs.event == 'push' }} + # See publish-wasm for why benchmark-action's own comment is + # disabled here and we post our own from this workflow. + alert-threshold: "110%" + fail-on-alert: false + summary-always: true + comment-always: false + comment-on-alert: false + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Post sticky PR comment + if: ${{ steps.meta.outputs.event == 'pull_request' && steps.meta.outputs.pr_number != '' }} + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # pin@v7 + env: + PR_NUMBER: ${{ steps.meta.outputs.pr_number }} + PR_HEAD_SHA: ${{ steps.meta.outputs.pr_head_sha }} + BENCH_NAME: Native perf (Linux x86_64) + MARKER: "" + DATA_PATH: ./benchmark-data-repository/bench/native/data.js + with: + script: | + const fs = require('fs'); + + // First-ever run: gh-pages doesn't exist yet, so + // benchmark-action skipped its local clone. Nothing to + // diff against — skip the comment gracefully rather than + // failing the job. Subsequent runs will have data. + if (!fs.existsSync(process.env.DATA_PATH)) { + core.warning(`${process.env.DATA_PATH} not found — likely the first run on this repo; skipping comment`); + return; + } + const raw = fs.readFileSync(process.env.DATA_PATH, 'utf8'); + const match = raw.match(/window\.BENCHMARK_DATA\s*=\s*(\{[\s\S]+\});?\s*$/); + if (!match) { + core.setFailed(`could not parse ${process.env.DATA_PATH}`); + return; + } + const data = JSON.parse(match[1]); + const history = data.entries[process.env.BENCH_NAME]; + if (!history || history.length === 0) { + core.warning(`no history yet for "${process.env.BENCH_NAME}"; skipping comment`); + return; + } + + const current = history[history.length - 1]; + const baseline = history.length > 1 ? history[history.length - 2] : null; + + const fmt = (v, unit) => `${Number(v).toLocaleString('en-US', { maximumFractionDigits: 0 })} ${unit}`; + const pctDelta = (cur, prev) => { + if (prev === undefined || prev === null || prev === 0) return 'n/a'; + const pct = ((cur - prev) / prev) * 100; + const sign = pct >= 0 ? '+' : ''; + const tag = pct > 10 ? ' ⚠️' : pct < -10 ? ' ⚡' : ''; + return `${sign}${pct.toFixed(1)}%${tag}`; + }; + + let body = `${process.env.MARKER}\n## ${process.env.BENCH_NAME}\n\n`; + body += '| Bench | Baseline | Current | Δ |\n|---|---|---|---|\n'; + for (const bench of current.benches) { + const prev = baseline?.benches.find((b) => b.name === bench.name); + body += `| \`${bench.name}\` | ${prev ? fmt(prev.value, prev.unit) : '—'} | ${fmt(bench.value, bench.unit)} | ${pctDelta(bench.value, prev?.value)} |\n`; + } + body += `\nBaseline: latest \`${data.repoUrl?.split('/').slice(-2).join('/') || 'gh-pages'}\` entry. Comparing PR head \`${(process.env.PR_HEAD_SHA || '').slice(0, 8)}\`. _cc @WiktorStarczewski on regressions > 10%._\n`; + + const prNumber = parseInt(process.env.PR_NUMBER, 10); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + }); + const existing = comments.find((c) => c.body && c.body.startsWith(process.env.MARKER)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); + } diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d724dba1aa..714adbe09e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,4 +1,4 @@ -# Performance regression tracking for miden-crypto. +# Performance regression tracking for miden-crypto — UNTRUSTED stage. # # Two jobs run on every PR + every push to `next`: # @@ -11,61 +11,34 @@ # runner. Tracks the same primitives at native speed for # comparison + regression detection. # -# Each bench job is paired with a trusted `publish-*` job that downloads -# its artifact and calls `benchmark-action/github-action-benchmark` with -# the write-capable token. The split is deliberate: the bench jobs run -# PR-controlled code (Rust, npm install, Node) and therefore MUST NOT -# hold credentials, so they run with `contents: read` and -# `persist-credentials: false`. The publish jobs check out the default -# branch (trusted ref) before running the action, so the PR's tree never -# reaches a step that has write scope. This is the same split miden-vm's -# non-regression workflows use. +# Both jobs run PR-controlled code (Rust, npm install, Node) and so MUST +# NOT hold any privileged token. This workflow runs on `pull_request` +# (not `pull_request_target`), declares `permissions: contents: read` at +# both workflow and job scope, and strips the persisted token from +# `actions/checkout`. The results are uploaded as artifacts and consumed +# by the separate `bench-publish.yml` workflow, which fires via +# `workflow_run` from a trusted base-repo context. # -# `benchmark-action/github-action-benchmark`: -# - Stores each metric over time on the `gh-pages` branch under -# `bench/` (separate from the existing `docs/` subdir used by docs.yml, -# so the two don't collide). -# - Posts a sticky PR comment showing the diff vs the latest `next` -# baseline. Header per job, so wasm and native each get their own -# comment that updates in place. -# - Alerts (writes a comment header) on regression > 10 %. Doesn't fail -# the workflow — GHA Linux runners share CPUs and the noise floor on -# hash benches is real. False-positive alerts are cheap to ignore; -# false-negative reverts are expensive. +# Why two workflows: GitHub Advanced Security's +# `actions/cache-poisoning/poisonable-step` rule flags any step under +# `pull_request_target` that executes PR-supplied code, because the +# implicit `ACTIONS_RUNTIME_TOKEN` always present in such jobs grants +# write access to the Actions cache. The split — untrusted bench in +# this workflow, trusted publish in `bench-publish.yml` triggered by +# `workflow_run` — is the GitHub Security Lab's documented pattern +# for "bench on PR, comment on PR, can't trust the PR's code." See +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ name: bench -# `pull_request_target` (not `pull_request`) so the publish job's -# `GITHUB_TOKEN` keeps `pull-requests: write` and `contents: write` on -# fork PRs. Under `pull_request`, GitHub silently strips both scopes -# from `GITHUB_TOKEN` for fork PRs regardless of what `permissions:` -# declares, which makes the sticky comment step crash with "Resource -# not accessible by integration" on every external contribution. The -# same pattern is used in web-sdk's `check-linked-client-pr.yml` for -# the same reason. -# -# SAFETY: `pull_request_target` is safe here because the bench jobs -# (the only steps that execute PR-controlled code) hold no privileged -# token. Each bench job declares per-job `permissions: contents: read`, -# checks out the PR head with `persist-credentials: false`, and -# doesn't reference any `secrets.*` — so even though the PR's tree is -# on disk, build.rs / npm postinstall / Node has nothing privileged to -# reach for. The trusted publish jobs check out the base ref (which is -# what `pull_request_target` defaults to anyway), so PR-supplied code -# never touches a step that has the write token. Under -# `pull_request_target` the workflow file itself is the base's, not -# the PR's, so a malicious PR can't edit the workflow logic that's run -# against them. on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review] + pull_request: push: branches: [next] workflow_dispatch: -# Default to read-only at workflow scope. Privileged scopes are granted -# per-job and only on the trusted publish jobs that do NOT execute any -# PR-controlled code. +# Read-only at workflow scope; per-job permissions narrow this further. +# `bench-publish.yml` is where the write scopes live. permissions: contents: read @@ -89,11 +62,6 @@ jobs: steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 with: - # Under `pull_request_target`, the default checkout is the - # BASE branch — but we need to benchmark the PR's code, so - # explicitly target the PR head SHA. `github.sha` is the - # fallback for push and workflow_dispatch events. - ref: ${{ github.event.pull_request.head.sha || github.sha }} # The default checkout writes the GHA token to `.git/config`, # which PR-controlled build scripts (build.rs, npm postinstall, # etc.) would then have access to. Strip it — this job has no @@ -138,7 +106,7 @@ jobs: # `pipefail` is load-bearing: the driver streams to a file via # a shell redirect, so a panicking driver would still leave a # partial `results.json` on disk; `set -e` ensures the step - # fails fast and the trusted publish job never sees a bad + # fails fast and the trusted publish workflow never sees a bad # baseline. set -euo pipefail node driver.mjs > results.json @@ -153,15 +121,35 @@ jobs: jq -r '.[] | "| \(.name) | \(.value | tostring) |"' results.json } >> "$GITHUB_STEP_SUMMARY" + - name: Write metadata for publish workflow + working-directory: miden-bench-wasm + # `workflow_run.pull_requests` is empty for fork PRs, so the + # trusted publish workflow can't get the PR number from the + # event payload. Stash the PR number + event + head SHA here so + # the publish workflow has somewhere to look. Note: this file + # is PR-author-controlled by being on the runner where PR code + # executed — the publish workflow MUST validate / sanity-check + # it rather than trust it blindly. + run: | + set -euo pipefail + jq -n \ + --arg event "${{ github.event_name }}" \ + --arg pr_number "${{ github.event.pull_request.number || '' }}" \ + --arg pr_head_sha "${{ github.event.pull_request.head.sha || '' }}" \ + '{event: $event, pr_number: $pr_number, pr_head_sha: $pr_head_sha}' \ + > metadata.json + - name: Upload raw results as artifact - # Picked up by the trusted `publish-wasm` job, which runs the - # benchmark-action against this file with write-scope token. Also - # keeps the full per-batch sample distribution for post-hoc - # analysis when an alert fires. + # Picked up by `bench-publish.yml` via `workflow_run`. The + # publish workflow runs in the base repo's trusted context with + # write-scope token; this artifact is the only thing crossing + # the trust boundary. uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # pin@v4 with: name: bench-wasm-results - path: miden-bench-wasm/results.json + path: | + miden-bench-wasm/results.json + miden-bench-wasm/metadata.json retention-days: 30 bench-native: @@ -173,9 +161,6 @@ jobs: steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 with: - # See `bench-wasm` for why both `ref:` and `persist-credentials` - # are set explicitly under `pull_request_target`. - ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Install Rust @@ -203,131 +188,28 @@ jobs: # propagates through the `tee` and fails the step. Without # it, only tee's exit status is observed and a broken bench # would silently produce an empty/partial `bench-native.txt` - # for the trusted publish job to ingest as a baseline. + # for the trusted publish workflow to ingest as a baseline. set -euo pipefail cargo bench --bench hash --bench word --bench transpose \ -p miden-crypto -- --output-format bencher 2>&1 \ | tee bench-native.txt + - name: Write metadata for publish workflow + # See `bench-wasm` for why we emit this. + run: | + set -euo pipefail + jq -n \ + --arg event "${{ github.event_name }}" \ + --arg pr_number "${{ github.event.pull_request.number || '' }}" \ + --arg pr_head_sha "${{ github.event.pull_request.head.sha || '' }}" \ + '{event: $event, pr_number: $pr_number, pr_head_sha: $pr_head_sha}' \ + > metadata.json + - name: Upload raw results as artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # pin@v4 with: name: bench-native-results - path: bench-native.txt + path: | + bench-native.txt + metadata.json retention-days: 30 - - publish-wasm: - name: Publish WASM bench - needs: bench-wasm - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write # github-action-benchmark pushes to gh-pages on `push to next` - pull-requests: write # for sticky PR comment on pull_request_target runs - steps: - # Explicitly check out the default branch (trusted ref). Under - # `pull_request_target` this would be the default anyway, but - # spelling it out keeps the invariant ("publish jobs never touch - # PR-supplied code") visible to anyone reading the workflow. - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 - with: - ref: ${{ github.event.repository.default_branch }} - # `benchmark-action/github-action-benchmark` reads its credential - # from the `github-token` input rather than `.git/config`, so the - # persisted token isn't needed. Strip it to satisfy zizmor's - # `artipacked` audit. - persist-credentials: false - - - name: Download bench results - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 - with: - name: bench-wasm-results - path: results - - - name: Track + alert on regression - uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # pin@v1.22.1 - with: - tool: customSmallerIsBetter - output-file-path: results/results.json - # Separate gh-pages subdir so this never collides with docs.yml's - # `destination_dir: docs` deploy (which lives at /docs/). - benchmark-data-dir-path: bench/wasm - gh-pages-branch: gh-pages - # Push baseline updates ONLY on push to next (not on PR runs — - # those compare against the existing baseline, they don't update it). - auto-push: ${{ github.event_name == 'push' }} - comment-on-alert: true - # Regression alert at 10 %. Run-to-run noise on GHA shared - # runners is real but accepting 15-20 % headroom would defeat - # the purpose: a 15 % regression IS a regression worth - # investigating, not the noise floor. Drive variance down - # (longer batches, more samples) to fit under this threshold - # rather than widening the threshold to fit measured noise. - # See README "Noise reduction" for the roadmap to tighten further. - alert-threshold: "110%" - fail-on-alert: false # warn via comment, don't fail the PR - summary-always: true # keep the diff comment fresh on each run - # `pull_request_target` (vs. `pull_request`) is what keeps the - # write token alive for fork PRs — see the header comment. - # The event name we match here is `pull_request_target` - # because that's what `github.event_name` resolves to under - # this trigger; the value is NOT `pull_request`. - comment-always: ${{ github.event_name == 'pull_request_target' }} - alert-comment-cc-users: '@WiktorStarczewski' - github-token: ${{ secrets.GITHUB_TOKEN }} - - publish-native: - name: Publish native bench - # Serialized after `publish-wasm` so the two never race on gh-pages. - # Both jobs `git fetch gh-pages → commit → push`, and on a `push to - # next` event both fire concurrently — whichever pushes second hits - # non-fast-forward and the metric for that job is lost. Running them - # in series costs ~20 s on push events and zero on PR runs (which - # don't push). Cheaper than the alternative (merging the publishers - # into one job) because it keeps the per-job permission scope - # narrow and lets each artifact pass/fail independently. - needs: [bench-native, publish-wasm] - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # pin@v4 - with: - ref: ${{ github.event.repository.default_branch }} - # `benchmark-action/github-action-benchmark` reads its credential - # from the `github-token` input rather than `.git/config`, so the - # persisted token isn't needed. Strip it to satisfy zizmor's - # `artipacked` audit. - persist-credentials: false - - - name: Download bench results - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4 - with: - name: bench-native-results - path: results - - - name: Track + alert on regression - uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # pin@v1.22.1 - with: - tool: cargo - output-file-path: results/bench-native.txt - benchmark-data-dir-path: bench/native - gh-pages-branch: gh-pages - auto-push: ${{ github.event_name == 'push' }} - comment-on-alert: true - # Same 10 % threshold as wasm — native benches are ns-scale and - # timer noise is proportionally larger, but criterion's own - # statistical pruning already handles much of the per-iteration - # variance. If 10 % is too tight in practice, the path forward - # is more iterations / `iai-callgrind` instruction-count - # benchmarking, NOT a wider threshold. - alert-threshold: "110%" - fail-on-alert: false - summary-always: true - # See `publish-wasm` and the header comment for why we match - # on `pull_request_target` rather than `pull_request`. - comment-always: ${{ github.event_name == 'pull_request_target' }} - alert-comment-cc-users: '@WiktorStarczewski' - github-token: ${{ secrets.GITHUB_TOKEN }}