Repository navigation
cherry-pick(release/v1.206): run a package with no project in the default project (#4630) #14319
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke RPA Skill Tests (Windows) | |
| concurrency: | |
| group: smoke-rpa-skills-${{ github.head_ref || github.ref }} | |
| # Only `synchronize` moves the head SHA; `opened` and `reopened` can fire on a | |
| # commit a run is already in flight for, and a cancelled run reports the | |
| # required context as `cancelled`. See docs/REQUIRED-CHECKS.md Rule 3a. | |
| cancel-in-progress: ${{ github.event.action == 'synchronize' }} | |
| on: | |
| # No `paths:` filter on purpose. This workflow is a required status | |
| # check; if `paths:` excluded a PR the workflow wouldn't run, no | |
| # check would be reported, and merge would block forever. Instead we | |
| # always trigger and let the `detect` job short-circuit via | |
| # `skip=true` — the gated job is then reported as skipped, which | |
| # GitHub counts as a pass for required checks. | |
| pull_request: | |
| workflow_dispatch: | |
| # Exempt the first-party `coder-eval` wheel from safe-chain's | |
| # minimum-package-age suppression so a same-day pin in | |
| # tests/.coder-eval-version still resolves — see smoke-skills.yml for the full | |
| # rationale. Workflow-level so any job added later inherits it. | |
| env: | |
| SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic | |
| jobs: | |
| detect: | |
| runs-on: uipath-ubuntu-latest | |
| name: Detect changed RPA skills | |
| outputs: | |
| task_globs: ${{ steps.detect.outputs.task_globs }} | |
| skip: ${{ steps.detect.outputs.skip }} | |
| steps: | |
| - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect changed Windows-tagged skills and map to test tasks | |
| id: detect | |
| env: | |
| DISABLE_SMOKE: ${{ vars.DISABLE_SMOKE }} | |
| run: | | |
| # Kill switch: repo variable DISABLE_SMOKE=true skips the smoke tests | |
| # (e.g. while coder-eval runs fail on expired registry tokens rather | |
| # than on skill regressions). Emitting skip=true keeps this required | |
| # check (and the gated one) green. Unset or any other value: run. | |
| if [ "$DISABLE_SMOKE" = "true" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "Smoke tests disabled (vars.DISABLE_SMOKE == 'true') — skipping" | |
| exit 0 | |
| fi | |
| BASE_REF="${{ github.base_ref }}" | |
| if [ -z "$BASE_REF" ]; then BASE_REF="main"; fi | |
| CHANGED=$(git diff --name-only origin/$BASE_REF...HEAD) | |
| # Discover the set of Windows-tagged task files (tag = `windows` in the | |
| # `tags:` line). Source of truth for "what runs on this Windows | |
| # workflow" — adding/removing a `windows` tag is the opt-in/out. | |
| WINDOWS_TASKS=$(grep -rEl '^tags:.*\bwindows\b' tests/tasks/ 2>/dev/null | sort -u) | |
| if [ -z "$WINDOWS_TASKS" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "No tasks tagged 'windows' found — skipping" | |
| exit 0 | |
| fi | |
| # Skills that own at least one windows-tagged task. Used to map | |
| # `skills/<X>/` changes to the right test set. | |
| WINDOWS_SKILLS=$(echo "$WINDOWS_TASKS" | sed -nE 's|^tests/tasks/([^/]+)/.*|\1|p' | sort -u) | |
| # Globs are relative to the `tests/` working dir of the smoke job. | |
| ALL_GLOBS=$(echo "$WINDOWS_TASKS" | sed 's|^tests/||' | tr '\n' ' ' | xargs) | |
| # Infra change → run every windows-tagged task. | |
| if echo "$CHANGED" | grep -qE '^tests/(experiments|_shared)/|^tests/[^/]+\.(py|yaml|toml)$|^\.github/workflows/smoke-rpa-skills\.yml$'; then | |
| echo "task_globs=$ALL_GLOBS" >> "$GITHUB_OUTPUT" | |
| echo "Running all Windows smoke tasks (test infrastructure changed)" | |
| exit 0 | |
| fi | |
| # Skill-scoped change → run only that skill's windows-tagged tasks. | |
| GLOBS="" | |
| for skill in $WINDOWS_SKILLS; do | |
| if echo "$CHANGED" | grep -qE "^(skills/${skill}/|tests/tasks/${skill}/)"; then | |
| for t in $WINDOWS_TASKS; do | |
| case "$t" in | |
| "tests/tasks/${skill}/"*) GLOBS="$GLOBS ${t#tests/}";; | |
| esac | |
| done | |
| fi | |
| done | |
| GLOBS=$(echo "$GLOBS" | xargs) | |
| if [ -z "$GLOBS" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "No Windows-tagged skill changes detected — skipping" | |
| else | |
| echo "task_globs=$GLOBS" >> "$GITHUB_OUTPUT" | |
| echo "Will test: $GLOBS" | |
| fi | |
| # The Windows smoke job installs coder-eval on the host (tempdir driver, | |
| # no Docker), so it can't read the agent image itself. Freeze the pinned | |
| # image's deps here on Linux and hand them over as an artifact. | |
| - name: Freeze coder-eval-agent deps as install constraints | |
| if: steps.detect.outputs.skip != 'true' | |
| run: | | |
| set -euo pipefail | |
| img="ghcr.io/uipath/coder-eval-agent:$(tr -d '[:space:]' < tests/.coder-eval-version)" | |
| echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| docker pull "$img" | |
| docker run --rm --entrypoint pip "$img" freeze --exclude-editable | grep -vE '^coder-eval| @ ' > "$RUNNER_TEMP/ce-constraints.txt" | |
| docker logout ghcr.io | |
| - name: Upload coder-eval constraints | |
| if: steps.detect.outputs.skip != 'true' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ce-constraints | |
| path: ${{ runner.temp }}/ce-constraints.txt | |
| retention-days: 1 | |
| smoke: | |
| needs: detect | |
| if: needs.detect.outputs.skip != 'true' | |
| runs-on: uipath-windows-latest | |
| # RPA smoke tasks spin up Helm on each `uip rpa` call (30–60s each). A | |
| # typical 3-task smoke run with cold Helm + three tasks runs ~25–40 | |
| # min; bump the ceiling so we don't cancel mid-task. | |
| timeout-minutes: 90 | |
| name: Run RPA skill smoke tests (Windows) | |
| # Disable uip CLI version-sync: tasks run in tempdir (no container) and | |
| # share one host ~/.uipath / global uip, so a `uip login status` re-pin | |
| # downgrades the CLI for later tasks. Mirrors daily.sh. | |
| env: | |
| UIPATH_CLI_DISABLE_VERSION_SYNC: "1" | |
| steps: | |
| - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 | |
| # Pin coder_eval to tests/.coder-eval-version so RPA smoke runs the same | |
| # engine as the skills smoke, ad-hoc, and nightly paths. shell: bash | |
| # because windows-latest defaults to PowerShell (no `tr`). | |
| - name: Resolve coder_eval pin | |
| id: ceref | |
| shell: bash | |
| run: echo "version=$(tr -d '[:space:]' < tests/.coder-eval-version)" >> "$GITHUB_OUTPUT" | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: '3.13' | |
| - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: '20' | |
| - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 | |
| with: | |
| dotnet-version: '8.0.x' | |
| # Host coder-eval CLI = pinned wheel (coder_eval feed + ml-packages for | |
| # deps). Windows RPA tasks run under the tempdir driver, so no agent image. | |
| # | |
| # claude-agent-sdk releases sometimes ship without a win_amd64 wheel | |
| # (0.2.144, 0.2.157, 0.2.160). uv then builds the sdist, which bundles no | |
| # claude.exe, and every task dies at agent_crash ("Claude Code not | |
| # found"). --only-binary makes the resolver skip any release lacking a | |
| # wheel for this platform and take the newest one that has it — no | |
| # per-release denylist, no upper cap. The check below fails fast if a | |
| # wheel ever arrives without the bundled binary. | |
| # | |
| # Constrained to the pinned agent image's frozen deps (frozen by the | |
| # detect job). claude-agent-sdk is dropped from them so --only-binary | |
| # keeps picking a release that ships a win_amd64 wheel. | |
| - name: Download coder-eval constraints | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: ce-constraints | |
| path: ${{ runner.temp }} | |
| - name: Install coder-eval | |
| shell: bash | |
| env: | |
| VERSION: ${{ steps.ceref.outputs.version }} | |
| run: | | |
| set -euo pipefail | |
| grep -v '^claude-agent-sdk' "$RUNNER_TEMP/ce-constraints.txt" > "$RUNNER_TEMP/ce-constraints-win.txt" | |
| uv pip install --system --only-binary claude-agent-sdk \ | |
| -c "$RUNNER_TEMP/ce-constraints-win.txt" "coder-eval==$VERSION" | |
| python3 -c "import pathlib, sys, claude_agent_sdk as s; exe = pathlib.Path(s.__file__).parent / '_bundled' / 'claude.exe'; print(f'claude-agent-sdk {s.__version__}: {exe}'); sys.exit(0 if exe.is_file() else f'::error::claude-agent-sdk {s.__version__} has no bundled claude.exe')" | |
| # The feed allows anonymous reads, so it is added without credentials. | |
| - name: Configure NuGet feed for Helm packages | |
| shell: bash | |
| run: | | |
| dotnet nuget add source \ | |
| "https://uipath.pkgs.visualstudio.com/Public.Feeds/_packaging/UiPath-Internal/nuget/v3/index.json" \ | |
| --name UiPath-Internal | |
| # Install from GitHub Packages `@dev` — the live prerelease train | |
| # published on every push to `cli/main`. Public npm mirrors the same | |
| # releases ~2 weeks later, so pulling `@latest` from public npm made | |
| # RPA-Windows tasks exercise a stale CLI that could miss freshly | |
| # renamed/added verbs. `@dev` keeps this runner in lockstep with the | |
| # Linux smoke image, which builds from the same `@dev` tag (see | |
| # smoke-skills.yml and tests/docker/Dockerfile). | |
| # | |
| # The `@uipath` scope is mapped to npm.pkg.github.com via a temp npmrc. | |
| # Token is GH_PACKAGES_TOKEN (read:packages only), never the repo-write | |
| # PAT: this workflow is `pull_request`-triggered, so anyone who can push | |
| # a branch to this public repo can read the secret it is handed. | |
| # | |
| # EVERY package is pinned to the explicit `@dev` dist-tag: the same feed | |
| # also carries a FROZEN `alpha` tag (1.197.0-alpha.*) and the public | |
| # `latest` line — never rely on `latest`/unspecified here, which could | |
| # resolve to a stale line. The npmrc is written and deleted inside this | |
| # step so the token never lingers on the runner. | |
| # | |
| # Tools are installed with raw `npm install -g @uipath/<tool>@dev`, NOT | |
| # `uip tools install`: on a `-dev` CLI the latter resolves the release | |
| # channel to `stable` and fails to find `-dev` tool builds (cli PR #2650). | |
| # uipcli's ToolManager then discovers them on disk under | |
| # `<npm-prefix>/@uipath/`, so `uip rpa …` / `uip rpa-legacy …` resolve | |
| # without triggering auto-install. Job-level UIPATH_CLI_DISABLE_VERSION_SYNC | |
| # keeps a `-dev` CLI from trying to re-align tools through the (stable-only) | |
| # channel path at runtime. | |
| # | |
| # `npm install -g` runs from a fresh tempdir so any future workspace-aware | |
| # `package.json` at the repo root cannot pull the install into a workspace. | |
| - name: Install uip CLI + RPA tools (GitHub Packages @dev) | |
| shell: bash | |
| env: | |
| GH_PACKAGES_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${GH_PACKAGES_TOKEN:-}" ]; then | |
| echo "::error::GH_PACKAGES_TOKEN is empty — cannot authenticate to the GitHub Packages @dev feed" | |
| exit 1 | |
| fi | |
| install_dir="$(mktemp -d)" | |
| cd "$install_dir" | |
| echo "Installing from $install_dir (workspace-isolated)" | |
| npmrc="$install_dir/.npmrc" | |
| { | |
| echo "@uipath:registry=https://npm.pkg.github.com/" | |
| echo "//npm.pkg.github.com/:_authToken=${GH_PACKAGES_TOKEN}" | |
| } > "$npmrc" | |
| export NPM_CONFIG_USERCONFIG="$npmrc" | |
| npm install -g @uipath/cli@dev | |
| uip --version | |
| for tool in @uipath/rpa-tool @uipath/rpa-legacy-tool; do | |
| echo "Installing ${tool}@dev" | |
| if ! npm install -g "${tool}@dev"; then | |
| echo "::error::${tool}@dev failed to install — no build on the dev train?" | |
| exit 1 | |
| fi | |
| done | |
| rm -f "$npmrc" | |
| unset NPM_CONFIG_USERCONFIG | |
| echo "--- Installed @uipath/* packages ---" | |
| npm ls -g --depth=0 2>/dev/null | grep '@uipath/' || true | |
| echo "--- uip tools list ---" | |
| uip tools list --output json | |
| # Pre-auth uip as a real licensed Studio user via Studio's own e2e | |
| # auth helper (vendored from Studio/.ci/helm-e2e/oauth-login.mjs). | |
| # Why this and not client_credentials: | |
| # * Helm's HelmFeatureGate runs TWO checks: sign-in state AND | |
| # license SKU entitlement (HelmLicenseSkuFeatureSourceService). | |
| # A licensed user account carries the Studio SKU and clears | |
| # both gates; an External App / client_credentials principal | |
| # typically doesn't. | |
| # * Studio Desktop scopes (StudioWebBackend, OrchestratorApiUserAccess, | |
| # LLMGateway, ...) are user-only — not requestable via | |
| # client_credentials grant at all. | |
| # * Studio engineers themselves use this exact script to drive | |
| # Helm e2e; we get free upkeep when login UI changes. | |
| # The script drives a headless Puppeteer browser through the | |
| # auth-code+PKCE flow, exchanges the code for tokens, then writes | |
| # the standard ~/.uipath/.auth file the JS CLI reads on disk | |
| # (refresh token included — much longer effective lifetime than | |
| # env-auth's ~1h). | |
| - name: Install Puppeteer (auth helper dep) | |
| shell: bash | |
| run: npm install --no-save puppeteer | |
| - name: Authenticate uip as licensed Studio user | |
| id: auth | |
| shell: bash | |
| env: | |
| AUTHORITY: https://alpha.uipath.com | |
| EMAIL: ${{ secrets.UIPATH_EMAIL }} | |
| PASSWORD: ${{ secrets.UIPATH_PASSWORD }} | |
| TENANT: ${{ vars.UIPATH_TENANT }} | |
| ORG: ${{ vars.UIPATH_ORG }} | |
| # Drop screenshots + DOM dumps here so we can upload them as an | |
| # artifact for post-mortem on auth failures. | |
| AUTH_DEBUG_DIR: ${{ github.workspace }}/auth-debug | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$AUTH_DEBUG_DIR" | |
| # Auth via puppeteer is occasionally flaky on alpha (org-select | |
| # SPA renders late, or the login form inputs intermittently lose | |
| # focus). Retry the login flow up to 3 times before giving up. | |
| for attempt in 1 2 3; do | |
| echo "--- Auth attempt $attempt ---" | |
| if node .github/scripts/uipath-oauth-login.mjs; then | |
| break | |
| fi | |
| if [ "$attempt" = "3" ]; then | |
| echo "::error::Auth failed after 3 attempts" | |
| exit 1 | |
| fi | |
| sleep 5 | |
| done | |
| # Smoke check the auth context is recognized. Failure here fails | |
| # the job — we'd rather not run smoke tasks against a broken | |
| # login state and chase mysterious sign-in errors per task. | |
| uip login status --output json | |
| # Always upload the auth-debug screenshots / dumps so we have a | |
| # post-mortem artifact even on success (baseline for comparison). | |
| - name: Upload auth-debug artifacts | |
| if: always() && steps.auth.outcome != 'skipped' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: auth-debug-${{ github.run_id }} | |
| path: ${{ github.workspace }}/auth-debug | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Pre-warm Helm (download NuGet package before tests) | |
| shell: bash | |
| run: | | |
| mkdir -p /tmp/helm-warmup && cd /tmp/helm-warmup | |
| uip rpa list-instances --output json 2>&1 || true | |
| taskkill //F //IM UiPath.Studio.Helm.exe 2>/dev/null || true | |
| # Model guard, deliberately its OWN step (mirror of smoke-skills.yml): the smoke | |
| # step below carries continue-on-error, so a guard inside it is swallowed and the | |
| # failure surfaces only as a downstream "no task.json" error. Fail here instead, | |
| # named, before the run starts. | |
| - name: Require an agent model | |
| shell: bash | |
| env: | |
| AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} | |
| run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"' | |
| - name: Run RPA smoke tests | |
| env: | |
| SKILLS_REPO_PATH: ${{ github.workspace }} | |
| # Route the agent through AWS Bedrock. The Anthropic workspace | |
| # hit its API usage limit (resets 2026-05-01), so DirectRoute | |
| # is blocked. Bedrock has a separate quota. | |
| API_BACKEND: bedrock | |
| AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| BEDROCK_MODEL: ${{ vars.BEDROCK_MODEL }} | |
| # The gate's agent model, passed on the command line rather than pinned in | |
| # experiments/smoke-windows.yaml (see smoke-skills.yml for why). No literal | |
| # fallback: an unset variable fails the guard step above rather than gating a | |
| # model nobody configured. | |
| AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }} | |
| working-directory: tests | |
| id: smoke | |
| shell: bash | |
| run: | | |
| # Run each task separately, killing Helm between tasks so one | |
| # task's stale Studio state doesn't leak into the next. Track | |
| # per-task exit codes so one failure doesn't abort the loop | |
| # but still propagates to the overall step outcome. | |
| shopt -s globstar nullglob | |
| overall_exit=0 | |
| for task in ${{ needs.detect.outputs.task_globs }}; do | |
| # TODO(rpa legacy): Bedrock deprecated Opus 4.6 mid-2026-04 | |
| # and Opus 4.1 IDs return 400 'invalid model identifier' | |
| # on the EU regional inference profiles we have access to. | |
| # Sonnet trips Bedrock's content filter on the full legacy | |
| # XAML baseline reliably (3/3 retries). Skip the task in | |
| # CI until we have a working Bedrock model for it; the | |
| # task YAML itself is still tracked for local runs and | |
| # for an eventual model fix. | |
| case "$task" in | |
| tasks/uipath-rpa/legacy/*) | |
| echo "::warning::Skipping $task — pending Bedrock Opus model fix" | |
| continue | |
| ;; | |
| esac | |
| echo "--- Killing leftover Helm/Studio processes ---" | |
| taskkill //F //IM UiPath.Studio.Helm.exe 2>/dev/null || true | |
| echo "--- Running: $task ---" | |
| # We previously overrode the legacy task to Opus on Bedrock | |
| # to dodge a content filter that tripped on the full legacy | |
| # XAML baseline. Opus 4.6 was deprecated on Bedrock, and | |
| # both 4-1 (un-versioned) and the suggested | |
| # claude-opus-4-1-20250805-v1:0 also returned 400 'invalid | |
| # model identifier' on the EU regional inference profiles | |
| # we have access to. Drop the override and rely on the | |
| # content-filter retry below to absorb intermittent blocks. | |
| for attempt in 1 2 3; do | |
| echo "--- $task attempt $attempt ---" | |
| if coder-eval run "$task" \ | |
| -e experiments/smoke-windows.yaml --model "$AGENT_MODEL" --tags smoke -j 1 -v; then | |
| break | |
| fi | |
| # Only retry on Bedrock content-filter ERRORs (not on real | |
| # task failures — those are real signal). Inspect the most | |
| # recent task.json and bail out if final_status is anything | |
| # other than ERROR with content-filter details. | |
| latest_json="$(ls -t runs/*/default/*/00/task.json 2>/dev/null | head -n 1 || true)" | |
| if [ -n "$latest_json" ] && LATEST_JSON="$latest_json" python3 -c "import json,os,sys; d=json.load(open(os.environ['LATEST_JSON'])); det=d.get('error_details') or {}; msg=json.dumps(det)+(d.get('error_message') or ''); sys.exit(0 if d.get('final_status')=='ERROR' and 'content filter' in msg.lower() else 2)"; then | |
| echo "::warning::$task: content-filter error on attempt $attempt — retrying" | |
| if [ "$attempt" = "3" ]; then | |
| overall_exit=1 | |
| fi | |
| continue | |
| fi | |
| overall_exit=1 | |
| break | |
| done | |
| done | |
| exit $overall_exit | |
| continue-on-error: true | |
| # Upload as early as possible so the report artifact is available | |
| # for post-mortem regardless of whether later gate/summary steps | |
| # crash. | |
| - name: Upload HTML / JSON eval report | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: eval-report-${{ github.run_id }} | |
| path: | | |
| tests/runs/*/experiment.html | |
| tests/runs/*/experiment.md | |
| tests/runs/*/experiment.json | |
| tests/runs/*/experiment.log | |
| tests/runs/*/*/variant.html | |
| tests/runs/*/*/variant.md | |
| tests/runs/*/*/variant.json | |
| tests/runs/**/task.html | |
| tests/runs/**/task.json | |
| tests/runs/**/task.log | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| - name: Summarize results | |
| if: always() | |
| working-directory: tests | |
| shell: bash | |
| run: | | |
| run_dir=$(ls -td runs/*/ 2>/dev/null | head -n 1) | |
| if [ -z "$run_dir" ]; then | |
| echo "::warning::No run directory found — nothing to summarize" | |
| exit 0 | |
| fi | |
| echo "Run directory: $run_dir" | |
| echo "" | |
| echo "## Task results" | |
| python - <<'PY' | |
| import json, pathlib | |
| # Skip runs/latest — it's a symlink/copy of the most recent | |
| # timestamped run dir, so including it double-counts that task. | |
| run_dirs = [d for d in sorted(pathlib.Path('runs').glob('*/')) | |
| if d.name != 'latest'] | |
| rows = [] | |
| for run_dir in run_dirs: | |
| for p in sorted(run_dir.rglob('task.json')): | |
| d = json.loads(p.read_text(encoding='utf-8')) | |
| lr = d.get('llm_review') or {} | |
| rows.append(( | |
| d.get('task_id', '?'), | |
| d.get('final_status', '?'), | |
| d.get('weighted_score'), | |
| lr.get('score'), | |
| )) | |
| if not rows: | |
| print(' (no task.json files found)') | |
| width = max((len(r[0]) for r in rows), default=4) | |
| for task_id, status, score, llm_score in rows: | |
| score_s = f"{score:.2f}" if isinstance(score, (int, float)) else ' —' | |
| llm_s = f"{llm_score:.2f}" if isinstance(llm_score, (int, float)) else ' —' | |
| print(f" {task_id:<{width}} status={status:<8} score={score_s} llm={llm_s}") | |
| PY | |
| echo "" | |
| echo "HTML report artifact: eval-report-${{ github.run_id }}" | |
| echo "Download under the workflow run's Artifacts section and open experiment.html." | |
| # Single global gate: each task must hit weighted_score >= 0.7. | |
| # Replaces both the per-criterion `pass_threshold` gate and the | |
| # LLM-reviewer threshold gate — `weighted_score` is the | |
| # deterministic aggregate of the YAML success_criteria, so one | |
| # uniform bar at 0.7 is enough to enforce quality without | |
| # re-tuning each criterion's threshold by hand. | |
| - name: Enforce per-task score threshold (>= 0.7) | |
| if: always() | |
| working-directory: tests | |
| shell: bash | |
| env: | |
| TASK_THRESHOLD: "0.7" | |
| run: | | |
| python - <<'PY' | |
| import json, os, pathlib, sys | |
| threshold = float(os.environ.get('TASK_THRESHOLD', '0.7')) | |
| # Skip runs/latest — it's a symlink/copy of the most recent | |
| # timestamped run dir, so including it double-counts that task. | |
| run_dirs = [d for d in sorted(pathlib.Path('runs').glob('*/')) | |
| if d.name != 'latest'] | |
| task_jsons = [] | |
| for run_dir in run_dirs: | |
| task_jsons.extend(sorted(run_dir.rglob('task.json'))) | |
| if not task_jsons: | |
| print('::error::No task.json files found — nothing to gate') | |
| sys.exit(1) | |
| failures, missing = [], [] | |
| for p in task_jsons: | |
| d = json.loads(p.read_text(encoding='utf-8')) | |
| tid = d.get('task_id', p.parent.name) | |
| score = d.get('weighted_score') | |
| if not isinstance(score, (int, float)): | |
| missing.append(tid) | |
| continue | |
| if score < threshold: | |
| failures.append((tid, score)) | |
| for tid in missing: | |
| print(f'::error::{tid}: missing weighted_score') | |
| for tid, score in failures: | |
| print(f'::error::{tid}: weighted_score {score:.2f} < {threshold}') | |
| fails = len(failures) + len(missing) | |
| total = len(task_jsons) | |
| print(f'Results: {total - fails}/{total} task(s) >= {threshold}') | |
| if fails: | |
| sys.exit(1) | |
| PY | |
| - name: Fail if smoke step was cancelled | |
| if: steps.smoke.outcome == 'cancelled' | |
| shell: bash | |
| run: exit 1 |