Repository navigation
110 lines (96 loc) · 4.91 KB
/
Copy pathci.yml
File metadata and controls
110 lines (96 loc) · 4.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
name: CI
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
test:
name: Run all test suites (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
# ── 1. Checkout ────────────────────────────────────────────────────────
- name: Checkout repository
uses: actions/checkout@v4
# ── 2. Python setup ────────────────────────────────────────────────────
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: pip
# ── 3. Install runtime dependencies ───────────────────────────────────
- name: Install runtime dependencies
run: pip install -r requirements.txt
# ── 4. Install test dependencies ───────────────────────────────────────
# pytest — test runner
# httpx — required internally by starlette.testclient.TestClient
# joblib — required for model loading/inference
# torch — CPU build to prevent ModuleNotFoundError in unit/parity tests without bloating Vercel bundle
- name: Install test dependencies
run: |
pip install pytest httpx joblib
pip install torch --index-url https://download.pytorch.org/whl/cpu
# ── 5. Unit tests ──────────────────────────────────────────────────────
# Tests tokenizer/slang_serializer.py round-trip.
# No network, no solver, no env vars needed.
- name: Run unit tests
run: pytest tests/unit/ -v --tb=short
env:
GROQ_API_KEY: ""
# ── 6. Integration tests ───────────────────────────────────────────────
# Tests POST /solve against the live Starlette app using TestClient.
# GROQ_API_KEY must be empty so FallbackSolver is always selected.
- name: Run integration tests
run: pytest tests/integration/ -v --tb=short
env:
GROQ_API_KEY: ""
# ── 7. Regression tests ────────────────────────────────────────────────
# Loads every .json fixture from tests/regression/fixtures/ and
# runs solver.solve() against it, comparing to the golden output.
# GROQ_API_KEY must be empty so FallbackSolver is always selected.
- name: Run regression tests
run: pytest tests/regression/ -v --tb=short
env:
GROQ_API_KEY: ""
# ── 8. Full suite run (all three together) ─────────────────────────────
# Runs all suites in one pytest process to catch any cross-suite
# singleton leakage that the per-suite runs would not detect.
# conftest.py's autouse fixture handles singleton reset between tests.
- name: Run full test suite
run: pytest tests/ -v --tb=short
env:
GROQ_API_KEY: ""
benchmark-integrity:
name: Verify benchmark dataset is not empty
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check checkpoint references have matching benchmarks
run: |
echo "--- Checking if checkpoints/final/best.pt is referenced ---"
REFS=$(grep -rl "checkpoints/final/best.pt" --include="*.py" --include="*.md" . || true)
if [ -n "$REFS" ]; then
echo "Found references to checkpoints/final/best.pt in:"
echo "$REFS"
echo ""
echo "--- Checking eval/benchmarks/ is not empty ---"
BENCHMARKS=$(find eval/benchmarks -name "*.json" 2>/dev/null | head -1)
if [ -z "$BENCHMARKS" ]; then
echo "FAIL: checkpoints/final/best.pt is referenced but eval/benchmarks/ has no .json files!"
echo "This will cause run_pipeline.py to silently report 0.0 accuracy."
echo "Run: python eval/generate_benchmarks.py"
exit 1
else
echo "OK: eval/benchmarks/ contains benchmark files."
find eval/benchmarks -name "*.json" -exec echo " - {}" \;
fi
else
echo "No references to checkpoints/final/best.pt found. Skipping check."
fi