diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..252b38f6 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,70 @@ +name: benchmarks + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: '23 3 * * *' # nightly ~03:23 UTC + workflow_dispatch: + +jobs: + bench-smoke: + name: Benchmark smoke (compile + run 1x) + runs-on: ubuntu-latest + defaults: + run: + working-directory: aide + steps: + - uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: aide/go.mod + cache-dependency-path: aide/go.sum + + - name: Download dependencies + run: go mod download + + # 1x each: proves every benchmark compiles and runs, cheap. This is the + # PR gate. NOTE: until the benchmark _test.go files land (dedicated PR), + # this passes vacuously (go test -bench=. with no benchmarks exits 0). + - name: Benchmark smoke test + run: make bench-smoke + + bench-measure: + name: Benchmark measurement + runs-on: ubuntu-latest + defaults: + run: + working-directory: aide + steps: + - uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: aide/go.mod + cache-dependency-path: aide/go.sum + + - name: Download dependencies + run: go mod download + + - name: Run benchmarks + run: | + go test -bench=. -benchmem -benchtime=200ms -count=3 \ + ./pkg/store/... ./pkg/survey/... ./pkg/grammar/... ./pkg/code/... ./pkg/memory/... \ + > bench-results.txt 2>&1 + tail -5 bench-results.txt + + # path is relative to GITHUB_WORKSPACE (repo root), not working-directory, + # so the file written inside aide/ is aide/bench-results.txt here. + - name: Upload benchmark results + uses: actions/upload-artifact@v7 + with: + name: go-benchmarks + path: aide/bench-results.txt + retention-days: 14 + if-no-files-found: error diff --git a/Makefile b/Makefile index c3c3c5f7..3fbb13c1 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ # make release-push VERSION=1.2.0 Bump to specific, commit, tag, and push # make hooks Install git hooks (lefthook) -.PHONY: release release-push build build-pprof build-web test test-ts test-go lint check-version check-release-needed hooks hooks-check +.PHONY: release release-push build build-pprof build-web test test-ts test-go lint check-version check-release-needed hooks hooks-check bench bench-smoke VERSION_FILES = package.json .claude-plugin/plugin.json .claude-plugin/marketplace.json .codex-plugin/plugin.json packages/opencode-plugin/package.json $(wildcard packages/aide-binary-*/package.json) @@ -127,6 +127,12 @@ test: hooks-check test-go test-ts test-go: $(MAKE) -C aide test +bench: + $(MAKE) -C aide bench + +bench-smoke: + $(MAKE) -C aide bench-smoke + test-ts: bunx vitest run --exclude='tests/memory-capture.test.ts' --exclude='dist/**' diff --git a/aide/Makefile b/aide/Makefile index 7009e034..ee1f552a 100644 --- a/aide/Makefile +++ b/aide/Makefile @@ -1,6 +1,6 @@ # aide Makefile -.PHONY: all build test clean lint fmt install install-plugin proto calibrate-tokens +.PHONY: all build test clean lint fmt install install-plugin proto calibrate-tokens bench bench-smoke bench-profile # Binary name BINARY=aide @@ -84,6 +84,38 @@ test-coverage: $(GOCMD) tool cover -html=coverage.out -o coverage.html @echo "Coverage report: coverage.html" +# Full benchmark run with allocations, 3 runs for stability. Packages are the +# hot-path areas that have (or will have) _bench_test.go files: store (bbolt + +# bleve code index and memory), survey (repo analysis), grammar (project scan), +# code (parse/tokenize), memory (scoring). +bench: + @echo "Running Go benchmarks..." + $(GOTEST) -bench=. -benchmem -benchtime=200ms -count=3 \ + ./pkg/store/... ./pkg/survey/... ./pkg/grammar/... ./pkg/code/... ./pkg/memory/... + +# Benchmark run with CPU + heap profiles written to pprof-out/. One go test +# invocation per package, since the profiler disallows writing profiles from +# multiple packages in a single run. Analyze with: +# go tool pprof pprof-out/store.cpu.out +bench-profile: + @mkdir -p pprof-out + @for pkg in store survey grammar code memory; do \ + echo "Profiling pkg/$$pkg ..."; \ + $(GOTEST) -bench=. -benchmem -benchtime=1s -count=1 \ + -cpuprofile=pprof-out/$$pkg.cpu.out \ + -memprofile=pprof-out/$$pkg.mem.out \ + ./pkg/$$pkg/... || exit 1; \ + done + @ls -la pprof-out + +# Fast check that every benchmark compiles and executes without error (one +# iteration each). Used in CI on every PR to catch broken benchmarks cheaply +# without paying for a meaningful measurement run. +bench-smoke: + @echo "Running Go benchmark smoke test (1 iteration each)..." + $(GOTEST) -run XXXNOTHING -bench=. -benchtime=1x -count=1 \ + ./pkg/store/... ./pkg/survey/... ./pkg/grammar/... ./pkg/code/... ./pkg/memory/... + lint: @echo "Running linter..." @if command -v $(GOLINT) > /dev/null; then \