diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..d3188d916f --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,279 @@ +name: Solver Time Benchmark + +# Comment-triggered: a maintainer (OWNER / MEMBER / COLLABORATOR) types +# `/bench` on a PR. The work is split across parallel jobs to cut wall-clock: +# +# resolve ─ gate + ack + resolve PR head/base SHAs +# │ +# ├─ build (matrix: pr | base) ── two self-contained choreo-cli binaries +# │ built concurrently, uploaded as artifacts +# │ +# └─ bench (matrix: one job per project variant) ── each downloads both +# binaries, benches ONLY its variant, renders that variant's +# SVGs, uploads a per-variant artifact +# │ +# collate ─ merges every per-variant artifact, pushes the rendered SVGs to a +# per-run CUSTOM ref `refs/bench-renders/` (a hidden ref, +# not a branch/tag, so it never shows in the Branches/Tags UI) and +# links them by commit SHA so they're viewable in the browser via +# GitHub's blob viewer (not just a ZIP), builds the comparison +# report, uploads the consolidated artifact, posts a fresh PR comment + +on: + issue_comment: + types: [created] + +permissions: + pull-requests: write + contents: write # collate pushes trajectory SVGs to a per-run refs/bench-renders/* custom ref + actions: read + +jobs: + resolve: + # Only run when: + # - the comment is on a PR (not a plain issue) + # - the body starts with "/bench" + # - the commenter has write access (gates external drive-by triggers) + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/bench') && + (github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'COLLABORATOR') + runs-on: ubuntu-slim + outputs: + head_sha: ${{ steps.pr.outputs.head_sha }} + base_sha: ${{ steps.pr.outputs.base_sha }} + head_repo: ${{ steps.pr.outputs.head_repo }} + variants: ${{ steps.variants.outputs.variants }} + steps: + - name: Acknowledge the trigger comment + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: rocket + + - name: Resolve PR head + base + id: pr + uses: actions/github-script@v7 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + core.setOutput('head_sha', pr.data.head.sha); + core.setOutput('base_sha', pr.data.base.sha); + core.setOutput('head_repo', pr.data.head.repo.full_name); + + - name: Checkout test-projects + enumerate script + uses: actions/checkout@v6 + with: + repository: ${{ steps.pr.outputs.head_repo }} + ref: ${{ steps.pr.outputs.head_sha }} + persist-credentials: false + sparse-checkout: | + test-projects + scripts/ci/enumerate-variants.sh + sparse-checkout-cone-mode: false + + - name: Enumerate project variants + id: variants + run: bash scripts/ci/enumerate-variants.sh + + # -------------------------------------------------------------------------- + # build choreo-cli for both PR and base concurrently + build: + needs: resolve + runs-on: ubuntu-24.04-arm + strategy: + fail-fast: false + matrix: + side: [pr, base] + steps: + - name: Checkout PR head + uses: actions/checkout@v6 + with: + repository: ${{ needs.resolve.outputs.head_repo }} + ref: ${{ needs.resolve.outputs.head_sha }} + fetch-depth: 0 + persist-credentials: false + + # For the base binary, overlay the base branch's Rust crates on top of + # the PR checkout (everything else — scripts, test-projects — is unused + # by the build itself). + - name: Overlay base sources + if: matrix.side == 'base' + run: | + git fetch origin ${{ needs.resolve.outputs.base_sha }} + git checkout ${{ needs.resolve.outputs.base_sha }} -- \ + src-core src-cli trajoptlib Cargo.lock Cargo.toml + + - name: Make GCC 14 the default toolchain + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 200 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 200 + + - name: Set up sccache + uses: mozilla-actions/sccache-action@v0.0.10 + + - name: Build choreo-cli (${{ matrix.side }}) + # if base is schema-incompatible continue anyways + continue-on-error: ${{ matrix.side == 'base' }} + run: cargo build --release -p choreo-cli + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_STARTUP_TIMEOUT: "600" + + - name: Upload binary + if: always() + uses: actions/upload-artifact@v7 + with: + name: cli-${{ matrix.side }} + path: target/release/choreo-cli + if-no-files-found: warn + + # -------------------------------------------------------------------------- + # parallelize bench for each project + # not parallelized further to avoid variance in HW causing noisy comparisons + bench: + needs: [resolve, build] + runs-on: ubuntu-24.04-arm + strategy: + fail-fast: false + matrix: + variant: ${{ fromJSON(needs.resolve.outputs.variants) }} + env: + VARIANT: ${{ matrix.variant }} + steps: + - name: Checkout PR head + uses: actions/checkout@v6 + with: + repository: ${{ needs.resolve.outputs.head_repo }} + ref: ${{ needs.resolve.outputs.head_sha }} + persist-credentials: false + + - name: Download PR binary + uses: actions/download-artifact@v7 + with: + name: cli-pr + path: bin-pr + + - name: Download base binary + id: dl_base + continue-on-error: true + uses: actions/download-artifact@v7 + with: + name: cli-base + path: bin-base + + - name: Make binaries executable + run: | + chmod +x bin-pr/choreo-cli + [ -f bin-base/choreo-cli ] && chmod +x bin-base/choreo-cli || true + + - name: Stage variant + run: | + mkdir -p tp-pr tp-base + cp -r "test-projects/$VARIANT" "tp-pr/$VARIANT" + cp -r "test-projects/$VARIANT" "tp-base/$VARIANT" + + - name: Run PR benchmark + continue-on-error: true + timeout-minutes: 60 + run: bash scripts/run-bench.sh "$PWD/bin-pr/choreo-cli" tp-pr out-pr "$VARIANT" + + + - name: Run base benchmark + continue-on-error: true # tolerate missing / schema-incompatible base + timeout-minutes: 60 + run: | + if [ -x bin-base/choreo-cli ]; then + bash scripts/run-bench.sh "$PWD/bin-base/choreo-cli" tp-base out-base "$VARIANT" + else + echo "no base binary available — skipping base run for $VARIANT" + fi + + - name: Render trajectory SVGs with linear acceleration coloring + run: bash scripts/ci/render-variant.sh + + - name: Upload per-variant artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: bench-${{ matrix.variant }} + path: | + out-pr/** + out-base/** + tp-pr/** + tp-base/** + if-no-files-found: warn + + # -------------------------------------------------------------------------- + collate: + needs: [resolve, bench] + runs-on: ubuntu-slim + steps: + - name: Checkout PR head + uses: actions/checkout@v6 + with: + repository: ${{ needs.resolve.outputs.head_repo }} + ref: ${{ needs.resolve.outputs.head_sha }} + persist-credentials: false + sparse-checkout: scripts + sparse-checkout-cone-mode: false + + - name: Merge per-variant artifacts + continue-on-error: true # if every bench job was skipped, still comment + uses: actions/download-artifact@v7 + with: + pattern: bench-* + path: merged + merge-multiple: true + + - name: Publish renders to a ref + id: renders + continue-on-error: true # a missing renders link must not block the comment + uses: actions/github-script@v7 + env: + HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + with: + script: | + const publish = require(`${process.env.GITHUB_WORKSPACE}/scripts/ci/publish-renders.cjs`) + await publish({ github, context, core }) + + - name: Build markdown report + env: + ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + RENDERS_URL: ${{ steps.renders.outputs.url }} + run: | + node scripts/bench-report.mjs \ + --pr merged/tp-pr --base merged/tp-base \ + --pr-reports merged/out-pr --base-reports merged/out-base \ + --artifact-url "$ARTIFACT_URL" \ + ${RENDERS_URL:+--renders-url "$RENDERS_URL"} \ + --commit "${HEAD_SHA:0:7}" \ + --out report.md + cat report.md + + - name: Upload consolidated bench artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: bench-output-${{ github.run_id }} + path: | + merged/out-pr/** + merged/out-base/** + merged/tp-pr/**/*.svg + merged/tp-base/**/*.svg + report.md + if-no-files-found: warn + + - name: Post benchmark comment on PR + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.issue.number }} + body-path: report.md diff --git a/.gitignore b/.gitignore index 241dc94a23..a5a7b7f5e8 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,6 @@ CMakeUserPresets.json build/ cli/ test-tmp*/ + +# Local PR-benchmark output (scripts/bench-local.sh) +bench-local/ diff --git a/Cargo.lock b/Cargo.lock index 00794a83b2..3c6867e2d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -565,6 +565,8 @@ version = "2026.0.3" dependencies = [ "choreo-core", "clap", + "serde", + "serde_json", "tracing", "tracing-subscriber", ] diff --git a/scripts/bench-local.sh b/scripts/bench-local.sh new file mode 100755 index 0000000000..2f59eb6891 --- /dev/null +++ b/scripts/bench-local.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# Local equivalent of .github/workflows/benchmark.yml — runs the bench pipeline +# against your working tree, optionally comparing to a base git ref, and writes +# everything (SVGs + reports + markdown) into ./bench-local/. +# +# Usage: +# scripts/bench-local.sh # PR-only run, no comparison +# scripts/bench-local.sh # full comparison vs (e.g. main) +# +# Requires: cargo and node (node also performs run-bench.sh's per-trajectory +# JSON merge). The base build uses `git worktree` so your current checkout +# stays untouched. +# +# Concurrency (locally we want to saturate the box, unlike CI's 1-variant-per- +# job matrix): variants are benched in parallel, and within each variant +# run-bench.sh runs that variant's trajectories in parallel too. Peak solver +# processes ≈ BENCH_VARIANT_JOBS × BENCH_TRAJ_JOBS, so size the product to your +# core count: +# BENCH_VARIANT_JOBS max variants in parallel (default: nproc/4, min 1) +# BENCH_TRAJ_JOBS trajectories per variant (run-bench.sh, default 4) +# Defaults give ≈ nproc concurrent solves. Example: +# BENCH_VARIANT_JOBS=8 BENCH_TRAJ_JOBS=4 scripts/bench-local.sh + +set -euo pipefail + +REPO_ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) +cd "$REPO_ROOT" + +BASE_REF=${1:-} +NPROC=$(nproc 2>/dev/null || echo 4) +VARIANT_JOBS=${BENCH_VARIANT_JOBS:-$(( NPROC / 4 > 0 ? NPROC / 4 : 1 ))} +OUT_ROOT="$REPO_ROOT/bench-local" +PR_TP="$OUT_ROOT/tp-pr" +BASE_TP="$OUT_ROOT/tp-base" +PR_OUT="$OUT_ROOT/out-pr" +BASE_OUT="$OUT_ROOT/out-base" +BASE_WORKTREE="$OUT_ROOT/base-worktree" + +if [ ! -d "$REPO_ROOT/test-projects" ]; then + echo "test-projects/ not found at repo root" >&2 + exit 1 +fi +for cmd in cargo node; do + command -v "$cmd" >/dev/null || { echo "missing required command: $cmd" >&2; exit 1; } +done + +# Bench every variant under $2 with $1, writing reports to $3, running up to +# $VARIANT_JOBS variants concurrently. Each variant is a separate run-bench.sh +# process (its own SHARD_ROOT, its own variant-prefixed report files — no +# cross-variant collision in the shared out dir). Per-variant output is teed to +# bench-local/logs/