Repository navigation
212 lines (193 loc) · 9.79 KB
/
Copy pathactivation-gate.yml
File metadata and controls
212 lines (193 loc) · 9.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
name: Skill Activation Gate
# When a SKILL.md frontmatter changes, run the activation eval restricted to
# that one skill's positives and fail if recall.yes drops more than DROP_PP
# (10pp — see tests/scripts/activation_gate.py) below the rounded 2026-06-17
# baseline. Cross-fire and negatives stay covered by the manual full run.
concurrency:
group: activation-gate-${{ github.head_ref || github.ref }}
# Allow-list, not deny-list: `synchronize` is the only event that arrives with
# a new head SHA, so it is the only one where cancelling is free. `opened`,
# `reopened` and `ready_for_review` all fire on a commit a run can already be
# in flight for, and a cancelled run reports the required context as
# `cancelled` — non-passing, exactly like `failure`. See
# docs/REQUIRED-CHECKS.md Rule 3a.
cancel-in-progress: ${{ github.event.action == 'synchronize' }}
on:
# No paths: filter — see smoke-skills.yml for the required-check rationale.
# ready_for_review is added so the gate fires the moment a PR exits draft;
# the gate job itself is then skipped on drafts (see `if:` below) to keep
# iteration-on-draft cheap.
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
workflow_dispatch:
permissions:
contents: read
# Exempt the first-party `coder-eval` wheel from safe-chain's
# minimum-package-age suppression so a same-day pin in
# tests/.coder-eval-version still resolves — see smoke-skills.yml for the full
# rationale. Workflow-level so any job added later inherits it.
env:
SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic
jobs:
detect:
runs-on: uipath-ubuntu-latest
outputs:
skills: ${{ steps.detect.outputs.skills }}
skip: ${{ steps.detect.outputs.skip }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with: { fetch-depth: 0 }
- id: detect
# base_ref is empty on workflow_dispatch — fall back to main so manual
# re-runs against a feature branch still resolve a sane merge-base.
# Block-style env: the inner `'main'` single quotes break flow-mapping
# parsing in strict YAML readers.
env:
BASE_REF: ${{ github.base_ref || 'main' }}
run: |
set -euo pipefail
BASE_SHA=$(git merge-base "origin/$BASE_REF" HEAD)
extract_fm() { awk '/^---$/{c++;if(c==2)exit;next} c==1{print}'; }
SKILLS=()
for f in $(git diff --name-only "$BASE_SHA"...HEAD | grep -E '^skills/[^/]+/SKILL\.md$' || true); do
# Skip deletions — the file is gone from HEAD; nothing to gate.
[ -f "$f" ] || continue
base_fm=$(git show "$BASE_SHA:$f" 2>/dev/null | extract_fm || true)
head_fm=$(extract_fm < "$f" || true)
[ "$base_fm" = "$head_fm" ] && continue
SKILLS+=("$(echo "$f" | cut -d/ -f2)")
done
if [ ${#SKILLS[@]} -eq 0 ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "No frontmatter changes — gate not triggered."
exit 0
fi
JSON=$(printf '%s\n' "${SKILLS[@]}" | jq -R . | jq -s -c .)
echo "skills=$JSON" >> "$GITHUB_OUTPUT"
echo "Will gate: ${SKILLS[*]}"
gate:
needs: detect
# Skip drafts to keep iteration cheap; the gate fires on transition to
# ready (via the ready_for_review event type above) and on every push to
# a ready PR. workflow_dispatch has no draft flag → the != 'true' check
# treats undefined as not-draft, so manual dispatch still runs the gate.
if: needs.detect.outputs.skip != 'true' && github.event.pull_request.draft != true
runs-on: uipath-ubuntu-latest
name: ${{ matrix.skill }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
skill: ${{ fromJSON(needs.detect.outputs.skills) }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# Pin coder_eval to tests/.coder-eval-version so the activation gate runs
# the same engine as the smoke, ad-hoc, and nightly paths.
- name: Resolve coder_eval pin
id: ceref
run: echo "version=$(tr -d '[:space:]' < tests/.coder-eval-version)" >> "$GITHUB_OUTPUT"
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: { python-version: '3.13' }
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
# Host coder-eval CLI = pinned wheel (coder_eval feed + ml-packages for
# deps), constrained to the dependency set frozen into the pinned agent
# image so activation resolves the same deps as smoke/ad-hoc/nightly.
# Activation itself runs under the tempdir driver -- the image is only
# the constraints source. Fork PRs get no secrets, so without the token
# this falls back to an unconstrained install rather than failing the
# required `Skill activation gate` context.
- name: Install coder-eval
env:
VERSION: ${{ steps.ceref.outputs.version }}
GHCR_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}
run: |
set -euo pipefail
constraints=()
if [ -n "${GHCR_TOKEN:-}" ]; then
img="ghcr.io/uipath/coder-eval-agent:$VERSION"
printf '%s' "$GHCR_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
docker pull "$img"
docker run --rm --entrypoint pip "$img" freeze --exclude-editable | grep -vE '^coder-eval| @ ' > "$RUNNER_TEMP/ce-constraints.txt"
docker logout ghcr.io
constraints=(-c "$RUNNER_TEMP/ce-constraints.txt")
else
echo "::warning::No GHCR token (fork PR?) — installing coder-eval without image constraints"
fi
uv pip install --system "${constraints[@]}" "coder-eval==$VERSION"
- name: Run activation gate
env:
# Pass matrix value via env (not direct ${{ }} interpolation in run:)
# to avoid shell expansion of any special chars in the skill name.
SKILL: ${{ matrix.skill }}
SKILLS_REPO_PATH: ${{ github.workspace }}
API_BACKEND: bedrock
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
AWS_REGION: ${{ vars.AWS_REGION }}
BEDROCK_MODEL: ${{ vars.BEDROCK_MODEL }}
# The agent model the gate measures (activation_gate.py passes it as --model,
# and exits non-zero when this is unset — no literal fallback). Its recall
# baselines are model-specific, so re-baseline when this moves.
AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
ACTIVATION_GATE_KEEP_DIR: ${{ runner.temp }}/activation-gate-tasks
run: python tests/scripts/activation_gate.py --skill "$SKILL"
# The rows' transcripts, so a miss can be read; uploaded only when the leg fails.
- name: Upload activation transcripts
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: activation-gate-${{ matrix.skill }}
path: ${{ runner.temp }}/activation-gate-tasks
if-no-files-found: ignore
retention-days: 7
# Fixed-name aggregator. The `gate` job above is named `${{ matrix.skill }}`,
# so its check name changes with every skill added or renamed — 61 distinct
# names in one quarter. Required-status-check contexts are literal strings, so
# a matrix leg can never be required: a name that doesn't run leaves the
# context pending forever. This job's name is constant, so require THIS one.
# It rolls up `detect` + every `gate` leg into a single pass/fail.
gate-summary:
name: Skill activation gate
needs: [detect, gate]
# Runs whatever `detect` and `gate` did, so the required context always
# reports; the rollup below is what turns their results into pass/fail.
# `!cancelled()` rather than `always()`, and the `${{ }}` wrapper is
# required (bare `!` starts a YAML tag) — docs/REQUIRED-CHECKS.md
# § Why `!cancelled()` and not `always()`.
if: ${{ !cancelled() }}
runs-on: uipath-ubuntu-latest
timeout-minutes: 5
steps:
- name: Roll up gate results
# Job results via env, not inline ${{ }} in run: — these are workflow-
# controlled values, but the repo keeps every interpolation out of the
# shell body regardless.
env:
DETECT_RESULT: ${{ needs.detect.result }}
GATE_RESULT: ${{ needs.gate.result }}
run: |
set -euo pipefail
echo "detect=$DETECT_RESULT gate=$GATE_RESULT"
# `needs.<matrix job>.result` is already the rollup of every leg:
# 'success' only when all legs succeeded, 'failure' when any failed.
# 'skipped' is the healthy no-op path and must pass.
#
# ALLOW-list the two good values rather than deny-listing the two bad
# ones. This is the repo's merge gate: an empty result, or a value
# GitHub adds later, must fail closed rather than fall through to
# "passed".
for result in "$DETECT_RESULT" "$GATE_RESULT"; do
case "$result" in
success|skipped) ;;
failure|cancelled) ;;
*) echo "::error::unrecognized job result '$result' — failing closed"; exit 1 ;;
esac
done
if [ "$DETECT_RESULT" = "failure" ] || [ "$DETECT_RESULT" = "cancelled" ]; then
echo "::error::activation gate could not determine which skills to gate (detect: $DETECT_RESULT)"
exit 1
fi
if [ "$GATE_RESULT" = "failure" ] || [ "$GATE_RESULT" = "cancelled" ]; then
echo "::error::at least one activation gate leg failed — see the per-skill jobs above"
exit 1
fi
echo "Activation gate passed."