Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
name: benchmarks

on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
- cron: '23 3 * * *' # nightly ~03:23 UTC
workflow_dispatch:

jobs:
bench-smoke:
name: Benchmark smoke (compile + run 1x)
runs-on: ubuntu-latest
defaults:
run:
working-directory: aide
steps:
- uses: actions/checkout@v7

- name: Setup Go
uses: actions/setup-go@v7
with:
go-version-file: aide/go.mod
cache-dependency-path: aide/go.sum

- name: Download dependencies
run: go mod download

# 1x each: proves every benchmark compiles and runs, cheap. This is the
# PR gate. NOTE: until the benchmark _test.go files land (dedicated PR),
# this passes vacuously (go test -bench=. with no benchmarks exits 0).
- name: Benchmark smoke test
run: make bench-smoke

bench-measure:
name: Benchmark measurement
runs-on: ubuntu-latest
defaults:
run:
working-directory: aide
steps:
- uses: actions/checkout@v7

- name: Setup Go
uses: actions/setup-go@v7
with:
go-version-file: aide/go.mod
cache-dependency-path: aide/go.sum

- name: Download dependencies
run: go mod download

- name: Run benchmarks
run: |
go test -bench=. -benchmem -benchtime=200ms -count=3 \
./pkg/store/... ./pkg/survey/... ./pkg/grammar/... ./pkg/code/... ./pkg/memory/... \
> bench-results.txt 2>&1
tail -5 bench-results.txt

# path is relative to GITHUB_WORKSPACE (repo root), not working-directory,
# so the file written inside aide/ is aide/bench-results.txt here.
- name: Upload benchmark results
uses: actions/upload-artifact@v7
with:
name: go-benchmarks
path: aide/bench-results.txt
retention-days: 14
if-no-files-found: error
8 changes: 7 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# make release-push VERSION=1.2.0 Bump to specific, commit, tag, and push
# make hooks Install git hooks (lefthook)

.PHONY: release release-push build build-pprof build-web test test-ts test-go lint check-version check-release-needed hooks hooks-check
.PHONY: release release-push build build-pprof build-web test test-ts test-go lint check-version check-release-needed hooks hooks-check bench bench-smoke

VERSION_FILES = package.json .claude-plugin/plugin.json .claude-plugin/marketplace.json .codex-plugin/plugin.json packages/opencode-plugin/package.json $(wildcard packages/aide-binary-*/package.json)

Expand Down Expand Up @@ -127,6 +127,12 @@ test: hooks-check test-go test-ts
test-go:
$(MAKE) -C aide test

bench:
$(MAKE) -C aide bench

bench-smoke:
$(MAKE) -C aide bench-smoke

test-ts:
bunx vitest run --exclude='tests/memory-capture.test.ts' --exclude='dist/**'

Expand Down
34 changes: 33 additions & 1 deletion aide/Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# aide Makefile

.PHONY: all build test clean lint fmt install install-plugin proto calibrate-tokens
.PHONY: all build test clean lint fmt install install-plugin proto calibrate-tokens bench bench-smoke bench-profile

# Binary name
BINARY=aide
Expand Down Expand Up @@ -84,6 +84,38 @@ test-coverage:
$(GOCMD) tool cover -html=coverage.out -o coverage.html
@echo "Coverage report: coverage.html"

# Full benchmark run with allocations, 3 runs for stability. Packages are the
# hot-path areas that have (or will have) _bench_test.go files: store (bbolt +
# bleve code index and memory), survey (repo analysis), grammar (project scan),
# code (parse/tokenize), memory (scoring).
bench:
@echo "Running Go benchmarks..."
$(GOTEST) -bench=. -benchmem -benchtime=200ms -count=3 \
./pkg/store/... ./pkg/survey/... ./pkg/grammar/... ./pkg/code/... ./pkg/memory/...

# Benchmark run with CPU + heap profiles written to pprof-out/. One go test
# invocation per package, since the profiler disallows writing profiles from
# multiple packages in a single run. Analyze with:
# go tool pprof pprof-out/store.cpu.out
bench-profile:
@mkdir -p pprof-out
@for pkg in store survey grammar code memory; do \
echo "Profiling pkg/$$pkg ..."; \
$(GOTEST) -bench=. -benchmem -benchtime=1s -count=1 \
-cpuprofile=pprof-out/$$pkg.cpu.out \
-memprofile=pprof-out/$$pkg.mem.out \
./pkg/$$pkg/... || exit 1; \
done
@ls -la pprof-out

# Fast check that every benchmark compiles and executes without error (one
# iteration each). Used in CI on every PR to catch broken benchmarks cheaply
# without paying for a meaningful measurement run.
bench-smoke:
@echo "Running Go benchmark smoke test (1 iteration each)..."
$(GOTEST) -run XXXNOTHING -bench=. -benchtime=1x -count=1 \
./pkg/store/... ./pkg/survey/... ./pkg/grammar/... ./pkg/code/... ./pkg/memory/...

lint:
@echo "Running linter..."
@if command -v $(GOLINT) > /dev/null; then \
Expand Down
Loading