Skip to content

feat(uipath-agents): Maestro Flow as a low-code agent tool #15429

feat(uipath-agents): Maestro Flow as a low-code agent tool

feat(uipath-agents): Maestro Flow as a low-code agent tool #15429

Workflow file for this run

name: Smoke Skill Tests
concurrency:
group: smoke-skills-${{ github.head_ref || github.ref }}
# Only `synchronize` moves the head SHA; `opened` and `reopened` can fire on a
# commit a run is already in flight for, and a cancelled run reports the
# required context as `cancelled`. See docs/REQUIRED-CHECKS.md Rule 3a.
cancel-in-progress: ${{ github.event.action == 'synchronize' }}
on:
# No `paths:` filter on purpose. This workflow is a required status
# check; if `paths:` excluded a PR the workflow wouldn't run, no
# check would be reported, and merge would block forever. Instead we
# always trigger and let the `detect` job short-circuit via
# `skip=true` — the gated job is then reported as skipped, which
# GitHub counts as a pass for required checks.
pull_request:
workflow_dispatch:
# Workflow-level, not per-job: every job that installs the pinned coder-eval
# wheel needs this, and a job added later inherits it automatically instead of
# silently regressing to "no version found".
#
# safe-chain's minimum-package-age check suppresses freshly published versions
# during metadata resolution, so a same-day `coder-eval` pin in
# tests/.coder-eval-version resolves to "no version found" and the install step
# fails. coder_eval is a first-party internal package from the Azure Artifacts
# feed, not public-registry supply-chain surface, so exclude it from the age
# check only. Every other safe-chain check still applies.
#
# `anthropic` is excluded for the same reason, transitively: coder-eval >=0.11.1
# pins anthropic>=1.0.0 (a fresh major), which the age gate suppresses — leaving
# the resolver with only anthropic<=0.123.0 and "requirements are unsatisfiable"
# until 1.0.0 ages out. The Anthropic SDK is the official first-party client, so
# exempting it from the age check (only) is the same low-risk call as coder-eval.
env:
SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic
jobs:
detect:
runs-on: uipath-ubuntu-latest
name: Detect changed skills
outputs:
task_globs: ${{ steps.detect.outputs.task_globs }}
skip: ${{ steps.detect.outputs.skip }}
untested_skills: ${{ steps.detect.outputs.untested_skills }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0
- name: Detect changed skills and map to test tasks
id: detect
env:
PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }}
MAX_SMOKE_TASKS: ${{ vars.SMOKE_MAX_TASKS || '40' }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.sha }}
DISABLE_SMOKE: ${{ vars.DISABLE_SMOKE }}
run: |
# Kill switch: repo variable DISABLE_SMOKE=true skips the smoke tests
# (e.g. while coder-eval runs fail on expired registry tokens rather
# than on skill regressions). Emitting skip=true keeps this required
# check (and the gated one) green. Unset or any other value: run.
if [ "$DISABLE_SMOKE" = "true" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "Smoke tests disabled (vars.DISABLE_SMOKE == 'true') — skipping"
exit 0
fi
BASE_REF="${{ github.base_ref }}"
BASE_REF="${BASE_REF:-main}"
CHANGED=$(git diff --name-only "origin/${BASE_REF}...HEAD")
# Per-PR smoke-task cap. No single PR runs more than MAX_SMOKE_TASKS
# smoke tasks: a `tests/_shared/` or smoke.yaml edit otherwise fans
# out to the whole tree (~250 tasks), which is too costly to pay on
# every PR. Over the cap we keep a deterministic pseudo-random sample
# seeded by the PR number (commit SHA on workflow_dispatch) so a
# re-run of the same commit selects the same set — a required check
# must not be a coin flip. Flat sample, NOT stratified by skill. The
# nightly full suite remains the coverage backstop for what's dropped.
SEED="$PR_NUMBER"
[ -z "$SEED" ] && SEED="$HEAD_SHA"
# apply_cap <space-separated glob/file tokens, relative to tests/>
# Echoes the token list unchanged when the eligible smoke-task count
# is at or under MAX_SMOKE_TASKS (preserves today's glob behavior).
# Over the cap, expands to smoke-tagged files, deterministically
# samples MAX_SMOKE_TASKS of them, and echoes explicit file paths
# (still relative to tests/). Smoke detection mirrors the repo's
# `^tags:` grep convention used elsewhere in this workflow.
apply_cap() {
shopt -s globstar nullglob
local tokens="$1" t f
local files=()
for t in $tokens; do
for f in tests/$t; do
[ -f "$f" ] || continue
grep -qE '^tags:.*\bsmoke\b' "$f" || continue
files+=("$f")
done
done
local uniq n
uniq=$(printf '%s\n' "${files[@]:-}" | grep -v '^$' | sort -u)
n=$(printf '%s\n' "$uniq" | grep -c . || true)
if [ "$n" -le "$MAX_SMOKE_TASKS" ]; then
echo "$tokens"
return
fi
echo "::notice::Smoke selection capped: $n eligible smoke task(s) -> $MAX_SMOKE_TASKS (seed=$SEED)" >&2
printf '%s\n' "$uniq" \
| while IFS= read -r p; do
printf '%s %s\n' "$(printf '%s%s' "$SEED" "$p" | sha1sum | cut -c1-16)" "$p"
done \
| sort | head -n "$MAX_SMOKE_TASKS" | cut -d' ' -f2- \
| sed 's|^tests/||' | xargs
}
# Windows-tagged tasks run on the Windows workflow
# (smoke-rpa-skills.yml). Helm/Studio only exists there, and
# the full lifecycle (`uip rpa` / `uip rpa-legacy` CLI) is
# what's being exercised — so we skip those tasks here.
# Source of truth: the `windows` tag in each task YAML, so adding
# a new Windows-only skill is just a tag, not a workflow edit.
WINDOWS_TASKS=$(grep -rEl '^tags:.*\bwindows\b' tests/tasks/ 2>/dev/null | sort -u)
# A skill is skipped whole only when it owns a Windows-tagged *smoke*
# task: this workflow runs `--tags smoke`, so a Windows-only e2e task
# (uipath-genome's build_component) must not take the skill's Linux
# smoke tasks down with it. Each Windows task file stays excluded on
# its own through WINDOWS_TASKS.
WINDOWS_SKILLS=$(for f in $WINDOWS_TASKS; do grep -qE '^tags:.*\bsmoke\b' "$f" && echo "$f"; done \
| sed -nE 's|^tests/tasks/([^/]+)/.*|\1|p' | sort -u)
# Opt-in benchmarks live alongside skill tests but are NOT smoke
# tests — they're large/expensive and run via the dashboard
# (`dashboard run --suite <name>`). Skip here so smoke stays cheap.
BENCHMARK_DIRS="activation"
# Single source of truth for "task file not smoke-eligible on Linux".
# Used by both the test-infra fan-out and the per-skill loop below.
# Accepts either a task-file path (matches the windows-task list
# exactly) or a `tests/tasks/<skill>/` directory prefix (matches the
# derived windows-skills / benchmark dirs).
is_excluded_path() {
local path="$1"
for skill in $BENCHMARK_DIRS; do
[[ "$path" == "tests/tasks/$skill/"* ]] && return 0
done
for skill in $WINDOWS_SKILLS; do
[[ "$path" == "tests/tasks/$skill/"* ]] && return 0
done
for t in $WINDOWS_TASKS; do
[[ "$path" == "$t" ]] && return 0
done
return 1
}
# If test infrastructure changed, run all smoke tests except
# the Windows-only ones and opt-in benchmarks.
#
# We only fan out for the smoke config (smoke.yaml) — other
# experiment files (e2e.yaml, activation.yaml, ...) drive their
# own jobs/dashboards and don't change smoke behavior, so editing
# them shouldn't trigger a 30-min full-suite run on every PR.
SKIP_INFRA_TRIGGER=false
echo "$PR_LABELS" | grep -qw 'ci:skip-infra-trigger' && SKIP_INFRA_TRIGGER=true
INFRA_PATTERN='^tests/docker/|^tests/fixtures/maestro-flow-sdk-smoke/'
INFRA_PATTERN+='|^tests/experiments/smoke\.yaml$|^tests/_shared/'
INFRA_PATTERN+='|^tests/[^/]+\.(py|yaml|toml)$|^tests/\.coder-eval-version$'
INFRA_PATTERN+='|^\.github/workflows/smoke-skills\.yml$'
if [ "$SKIP_INFRA_TRIGGER" = "false" ] && echo "$CHANGED" | grep -qE "$INFRA_PATTERN"; then
shopt -s globstar nullglob
GLOBS=""
for f in tests/tasks/**/*.yaml; do
is_excluded_path "$f" && continue
GLOBS="$GLOBS ${f#tests/}"
done
GLOBS=$(echo "$GLOBS" | xargs)
GLOBS=$(apply_cap "$GLOBS")
if [ -z "$GLOBS" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "No non-Windows smoke tasks to run."
else
echo "task_globs=$GLOBS" >> "$GITHUB_OUTPUT"
echo "Running non-Windows smoke tests (test infrastructure changed; capped at $MAX_SMOKE_TASKS)"
fi
exit 0
fi
# Skill SOURCE changes → run that skill's FULL smoke suite. The
# skill's behavior may have changed, so broad coverage is warranted.
SRC_SKILLS=$(echo "$CHANGED" | grep '^skills/' | sed 's|skills/\([^/]*\)/.*|\1|' | sort -u | grep -v '^$' || true)
# Test-only changes → run just the affected task file(s), NOT the
# whole skill. Adding or editing a test must not re-run every other
# smoke task for that skill when the skill itself is unchanged.
# Resolve each changed path under tests/tasks/ to its owning task YAML:
# - a changed *.yaml IS the task
# - a changed fixture / check-script resolves to the nearest
# ancestor directory that holds a *.yaml (the leaf task dir)
resolve_task_yaml() {
local path="$1"
if [[ "$path" == *.yaml ]]; then
echo "$path"
return
fi
local d
d=$(dirname "$path")
while [[ "$d" == tests/tasks/* ]]; do
local found
found=$(find "$d" -maxdepth 1 -name '*.yaml' 2>/dev/null)
if [ -n "$found" ]; then
echo "$found"
return
fi
d=$(dirname "$d")
done
}
# A changed non-YAML path can sit under a directory with no *.yaml
# anywhere above it up to tests/tasks/<skill>/ — a family-level
# _shared/ (grading helpers, reached only via $REFERENCE_DIR) or a
# family-level _setup/ (pre_run/post_run tooling shared by several
# leaf tasks), neither of which is itself a task directory.
# resolve_task_yaml() can't map those to one owning task, so such a
# change instead falls back to that skill's FULL smoke suite —
# otherwise it silently selected nothing at all.
CHANGED_TASKS=""
FALLBACK_SKILLS=""
while IFS= read -r path; do
[ -z "$path" ] && continue
found_any=""
for y in $(resolve_task_yaml "$path"); do
CHANGED_TASKS="$CHANGED_TASKS"$'\n'"$y"
found_any="1"
done
if [ -z "$found_any" ]; then
case "$path" in
*/_shared/*|*/_setup/*)
skill=$(echo "$path" | sed -nE 's|^tests/tasks/([^/]+)/.*|\1|p')
[ -n "$skill" ] && FALLBACK_SKILLS="$FALLBACK_SKILLS"$'\n'"$skill"
;;
esac
fi
done < <(echo "$CHANGED" | grep '^tests/tasks/' || true)
CHANGED_TASKS=$(echo "$CHANGED_TASKS" | sort -u | grep -v '^$' || true)
FALLBACK_SKILLS=$(echo "$FALLBACK_SKILLS" | sort -u | grep -v '^$' || true)
if [ -n "$FALLBACK_SKILLS" ]; then
echo "Falling back to full smoke suite for (changed _shared/_setup file with no owning task yaml): $(echo "$FALLBACK_SKILLS" | xargs)"
SRC_SKILLS=$(echo "$SRC_SKILLS"$'\n'"$FALLBACK_SKILLS" | sort -u | grep -v '^$' || true)
fi
if [ -z "$SRC_SKILLS" ] && [ -z "$CHANGED_TASKS" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "No skill or test changes detected — skipping smoke tests"
exit 0
fi
GLOBS=""
UNTESTED=""
# 1) Whole-skill globs for skills whose SOURCE changed.
# Skip skills whose tasks are all `windows`-tagged (they run on
# smoke-rpa-skills.yml) or are opt-in benchmarks.
for skill in $SRC_SKILLS; do
if echo "$WINDOWS_SKILLS" | grep -qw "$skill"; then
echo "Skipping $skill (runs on Windows — see smoke-rpa-skills.yml)"
continue
fi
if echo "$BENCHMARK_DIRS" | grep -qw "$skill"; then
echo "Skipping $skill (opt-in benchmark — runs via dashboard --suite $skill)"
continue
fi
if [ -d "tests/tasks/$skill" ]; then
GLOBS="$GLOBS tasks/$skill/**/*.yaml"
echo "Will test (skill source changed): $skill"
else
if [ -n "$UNTESTED" ]; then
UNTESTED="$UNTESTED, $skill"
else
UNTESTED="$skill"
fi
echo "No tests for: $skill (skipping)"
fi
done
# 2) Individual task files for test-only changes. Skip any task whose
# skill is already covered by a whole-skill glob above, and any
# Windows-only / benchmark path (those run elsewhere).
for task in $CHANGED_TASKS; do
skill=$(echo "$task" | sed -nE 's|^tests/tasks/([^/]+)/.*|\1|p')
if echo "$SRC_SKILLS" | grep -qw "$skill"; then
continue # already covered by the whole-skill glob
fi
# NOTE: is_excluded_path() internally loops `for t in $WINDOWS_TASKS`
# (non-local), so don't name this loop variable `t` — it would be
# clobbered by the call below.
if is_excluded_path "$task"; then
echo "Skipping $task (Windows-only or benchmark — not a Linux smoke task)"
continue
fi
GLOBS="$GLOBS ${task#tests/}"
echo "Will test (task changed): ${task#tests/}"
done
GLOBS=$(echo "$GLOBS" | xargs)
GLOBS=$(apply_cap "$GLOBS")
if [ -n "$UNTESTED" ]; then
echo "untested_skills=$UNTESTED" >> "$GITHUB_OUTPUT"
fi
if [ -z "$GLOBS" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
if [ -n "$UNTESTED" ]; then
echo "::warning::Changed skills have no smoke tests: $UNTESTED"
else
echo "No smoke-eligible tasks to run."
fi
else
echo "task_globs=$GLOBS" >> "$GITHUB_OUTPUT"
fi
warn-untested:
needs: detect
if: needs.detect.outputs.untested_skills != ''
runs-on: uipath-ubuntu-latest
name: Warn about untested skills
permissions:
pull-requests: write
steps:
- uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
script: |
const skills = '${{ needs.detect.outputs.untested_skills }}';
const body = `⚠️ **Smoke test coverage gap** — the following changed skills have no tests under \`tests/tasks/\`:\n\n${skills.split(', ').map(s => '- `' + s + '`').join('\n')}\n\nConsider adding smoke tests before merging.`;
// Avoid duplicate comments on repeated pushes
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.includes('Smoke test coverage gap'));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body,
});
}
e2e:
needs: detect
if: needs.detect.outputs.skip != 'true'
runs-on: uipath-ubuntu-latest
# 18+ smoke tasks at ~1–2 min each + env setup. 60 min was getting
# cancelled when the matrix grew + LLM reviewer was retrying on quota
# exhaustion. 120 min covers worst-case reviewer-retry storms without
# masking real hangs.
timeout-minutes: 120
name: Run skill smoke tests
# Disable uip CLI version-sync: a task's `uip login status` re-pins
# ~/.uipath/config.json to a stale line and, via smoke.yaml's shared
# ~/.uipath mount, downgrades the CLI for later tasks. Mirrors daily.sh.
env:
UIPATH_CLI_DISABLE_VERSION_SYNC: "1"
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# Single source of truth for the coder_eval version: the pin in
# tests/.coder-eval-version. Consumed as the wheel version (host CLI) and
# the agent image tag (sandbox) below -- no coder_eval source checkout.
- name: Resolve coder_eval pin
id: ceref
run: echo "version=$(tr -d '[:space:]' < tests/.coder-eval-version)" >> "$GITHUB_OUTPUT"
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '20'
- name: Verify Docker is available
run: docker info > /dev/null
# Pull the pinned agent image from GHCR (never built from source); the
# skills smoke image extends it below, and its frozen deps constrain the
# host install. Needs read:packages only —
# GH_PACKAGES_TOKEN, never the repo-write PAT: this workflow is
# `pull_request`-triggered, so anyone who can push a branch to this
# public repo can read the secret it is handed.
- name: Pull pinned coder-eval-agent image
run: |
echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
docker pull ghcr.io/uipath/coder-eval-agent:${{ steps.ceref.outputs.version }}
# Host coder-eval CLI = pinned wheel from the Azure Artifacts `coder_eval`
# feed; its UiPath deps resolve from the ml-packages feed (both as extra
# indexes). No source checkout. Constrained to the image's frozen deps so
# host and container resolve the same versions. The [codex] extra is
# required even though the agent itself runs inside the container: the
# docker driver imports the selected agent class on the HOST before
# dispatching work.
- name: Install coder-eval
env:
VERSION: ${{ steps.ceref.outputs.version }}
run: |
set -euo pipefail
docker run --rm --entrypoint pip "ghcr.io/uipath/coder-eval-agent:$VERSION" freeze --exclude-editable | grep -vE '^coder-eval| @ ' > "$RUNNER_TEMP/ce-constraints.txt"
uv pip install --system --prerelease=allow -c "$RUNNER_TEMP/ce-constraints.txt" "coder-eval[codex,litellm]==$VERSION"
# Build custom smoke image extending coder-eval-agent with @uipath/cli,
# every whitelisted tool plugin from GitHub Packages @dev, and the latest
# published @uipath/maestro-builder-sdk (see tests/docker/Dockerfile). Every push to
# `cli/main` publishes a `-dev.<run>` prerelease under the `dev` dist-tag
# (cli PR #2650 renamed the old `alpha` tag → `dev` on 2026-06-26), so
# smoke tracks main — not stale `latest` from public npm (which lags by ~2
# weeks).
- name: Build skills smoke Docker image
env:
DOCKER_BUILDKIT: "1"
NPM_AUTH_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}
FLOW_SDK_PKG_NAME: ${{ vars.FLOW_SDK_PKG_NAME }}
run: |
# The image's ARG default is the post-rename package name; override it
# only when a repo var pins a different one. The pre-rename baseline
# needs BOTH this and FLOW_SDK_VERSION (see tests/docker/Dockerfile) —
# pinning the version alone asks for a version the new name never had.
pkg_name_arg=()
if [ -n "${FLOW_SDK_PKG_NAME:-}" ]; then
pkg_name_arg=(--build-arg "FLOW_SDK_PKG_NAME=${FLOW_SDK_PKG_NAME}")
fi
docker build \
--build-arg CODER_EVAL_IMAGE=ghcr.io/uipath/coder-eval-agent:${{ steps.ceref.outputs.version }} \
--secret id=npm_auth_token,env=NPM_AUTH_TOKEN \
--build-arg FLOW_SDK_VERSION="${{ vars.FLOW_SDK_VERSION || 'latest' }}" \
--build-arg CONNECTORS_LIBRARY_VERSION="${{ vars.CONNECTORS_LIBRARY_VERSION || 'latest' }}" \
${pkg_name_arg[@]+"${pkg_name_arg[@]}"} \
-t skills-image:latest \
-f tests/docker/Dockerfile \
.
# Authenticate as the existing Alpha bot user. Agent guardrail discovery
# is served by agentsruntime_ and requires user-only scopes such as
# StudioWebBackend; an External App client_credentials token can report a
# valid login while still receiving 401 from the guardrail endpoints.
#
# Reuse the same ROPC helper as the nightly workflow. It writes the
# standard ~/.uipath/.auth file that the smoke experiment mounts into
# every Docker sandbox.
- name: Mint UiPath Alpha bot token (ROPC)
env:
UIPATH_URL: https://alpha.uipath.com
UIPATH_ORGANIZATION_NAME: ${{ vars.UIPATH_ORG_NAME }}
UIPATH_ORGANIZATION_ID: ${{ vars.UIPATH_ORG_ID }}
UIPATH_TENANT_NAME: ${{ vars.UIPATH_TENANT_NAME }}
UIPATH_TENANT_ID: ${{ vars.UIPATH_TENANT_ID }}
CLIENT_ID: ${{ vars.UIPATH_ROPC_CLIENT_ID }}
CLIENT_SECRET: ${{ secrets.UIPATH_ROPC_CLIENT_SECRET }}
CE_USERNAME: ${{ vars.UIPATH_BOT_USERNAME }}
CE_PASSWORD: ${{ secrets.UIPATH_BOT_PASSWORD }}
run: |
set -euo pipefail
bash .github/scripts/refresh-auth.sh
# coder-eval 0.11+ drops DAC_OVERRIDE/DAC_READ_SEARCH from task
# containers (reference anti-cheat), so container-root can no longer
# read the host-owned ~/.uipath extra_mount by capability. Widen it
# recursively via the "other" bits so the mounted .auth file (mode
# 600) is readable — mirrors coder-eval's own grant_container_access.
chmod -R o+rwX "$HOME/.uipath"
# Fail once at setup, with a direct error, rather than letting every
# guardrail task discover an unusable auth context independently.
docker run --rm \
-v "$HOME/.uipath:/.uipath:rw" \
--entrypoint /bin/sh \
skills-image:latest -lc '
set -eu
uip login status --output json
uip agent guardrails list --output json
'
# Model guard, deliberately its OWN step: "Run smoke tests" below carries
# continue-on-error, and the threshold steps treat zero tasks as a pass, so a
# guard inside the smoke step is swallowed and the gate goes green having
# evaluated nothing. Fail here instead, before any task runs.
- name: Require an agent model
env:
AGENT_MODEL: ${{ vars.CODEX_MODEL }}
run: ': "${AGENT_MODEL:?unset — set the CODEX_MODEL repo variable}"'
# coder-eval 0.11 isolates $TASK_DIR to the leaf task folder, so graders
# can no longer import the group-level _shared/ helper by walking up the
# tree. Co-locate a copy of each group's _shared into its task dirs first.
- name: Stage _shared grader helpers
run: bash tests/scripts/stage_shared.sh
- name: Run smoke tests
env:
SKILLS_REPO_PATH: ${{ github.workspace }}
# Route the agent through AWS Bedrock. The Anthropic workspace
# hit its API usage limit (resets 2026-05-01), so DirectRoute
# is blocked. Bedrock has a separate quota.
API_BACKEND: bedrock
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
AWS_REGION: ${{ vars.AWS_REGION }}
BEDROCK_MODEL: ${{ vars.BEDROCK_MODEL }}
TASK_PARALLELISM: ${{ vars.TASK_PARALLELISM || '1' }}
# Codex auth: the key + the endpoint the codex binary talks to. Same secrets
# the nightly and run-coder-eval.yml use.
CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
CODEX_BASE_URL: ${{ vars.CODEX_BASE_URL }}
# The gate's agent model, passed explicitly on the command line rather than
# pinned in experiments/smoke.yaml — one repo variable moves every gate here,
# and the experiment file stays harness-agnostic (the nightly runs the same
# file under claude/antigravity with their own models). No literal fallback:
# gating on a model nobody configured reports a verdict about the wrong thing,
# so an unset variable fails the guard step above. NOT $BEDROCK_MODEL: that
# is the evaluation-side model (llm_judge + the simulated user), which stays
# on Claude on purpose — the grader must not move with the agent.
AGENT_MODEL: ${{ vars.CODEX_MODEL }}
working-directory: tests
id: smoke
run: |
shopt -s globstar nullglob
echo "Running: coder-eval run ${{ needs.detect.outputs.task_globs }} --type codex --model $AGENT_MODEL --tags smoke -j $TASK_PARALLELISM"
coder-eval run ${{ needs.detect.outputs.task_globs }} \
-e experiments/smoke.yaml \
--type codex \
--model "$AGENT_MODEL" \
-j "$TASK_PARALLELISM" -v \
--run-dir /tmp/runs \
--tags smoke
continue-on-error: true
- name: Fix permissions on /tmp/runs
if: always()
run: sudo chown -R $(id -u):$(id -g) /tmp/runs && sudo chmod -R 755 /tmp/runs 2>/dev/null || true
- name: Summarize results
if: always()
run: |
# Find the most recent run directory (tests may not have produced
# one on catastrophic install failures).
run_dir=$(ls -td /tmp/runs/*/ 2>/dev/null | head -n 1)
if [ -z "$run_dir" ]; then
echo "::warning::No run directory found — nothing to summarize"
exit 0
fi
echo "Run directory: $run_dir"
echo ""
echo "## Task results"
python - <<'PY'
import json
import pathlib
run_dir = sorted(pathlib.Path('/tmp/runs').glob('*/'))[-1]
rows = []
for p in sorted(run_dir.rglob('task.json')):
d = json.loads(p.read_text(encoding='utf-8'))
lr = d.get('llm_review') or {}
rows.append((
d.get('task_id', '?'),
d.get('final_status', '?'),
d.get('weighted_score'),
lr.get('score'),
))
if not rows:
print(' (no task.json files found)')
width = max((len(r[0]) for r in rows), default=4)
for task_id, status, score, llm_score in rows:
score_s = f"{score:.2f}" if isinstance(score, (int, float)) else ' —'
llm_s = f"{llm_score:.2f}" if isinstance(llm_score, (int, float)) else ' —'
print(f" {task_id:<{width}} status={status:<8} score={score_s} llm={llm_s}")
PY
echo ""
echo "HTML report artifact: eval-report-${{ github.run_id }}"
echo "Download under the workflow run's Artifacts section and open experiment.html."
- name: Redact secrets from eval report
if: always()
env:
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
run: |
python - <<'PY'
import os
import pathlib
names = (
"UIPATH_CLI_AUTH_TOKEN",
"AWS_BEARER_TOKEN_BEDROCK",
)
values = [os.environ[name] for name in names if len(os.environ.get(name, "")) >= 8]
if not values:
raise SystemExit(0)
suffixes = {".html", ".json", ".log", ".md"}
for path in pathlib.Path("/tmp/runs").rglob("*"):
if not path.is_file() or path.suffix not in suffixes:
continue
text = path.read_text(encoding="utf-8", errors="ignore")
redacted = text
for value in values:
redacted = redacted.replace(value, "***")
if redacted != text:
path.write_text(redacted, encoding="utf-8")
PY
- name: Clean up artifacts (remove .venv and node_modules)
if: always()
run: |
find /tmp/runs -type d \( -name .venv -o -name node_modules \) -path "*/artifacts/*" -exec rm -rf {} + || true
- name: Upload HTML / JSON eval report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: eval-report-${{ github.run_id }}
# Use ** so the pattern still matches the replicate-index segment
# that coder_eval adds to per-task run dirs (/tmp/runs/<ts>/<variant>/<task>/<NN>/).
path: |
/tmp/runs/*/experiment.html
/tmp/runs/*/experiment.md
/tmp/runs/*/experiment.json
/tmp/runs/*/experiment.log
/tmp/runs/*/*/variant.html
/tmp/runs/*/*/variant.md
/tmp/runs/*/*/variant.json
/tmp/runs/**/task.html
/tmp/runs/**/task.json
/tmp/runs/**/task.log
/tmp/runs/**/artifacts
if-no-files-found: warn
retention-days: 14
# Quality gate: fail the job if any task's LLM reviewer score is
# below 0.7, independent of deterministic criteria. The reviewer
# catches skill-adherence regressions that file-exists /
# file-contains checks miss (e.g. agent called the wrong subcommand
# but still produced the right output shape). 0.7 (not 0.8) gives
# the qualitative verdict room to dock for minor process issues the
# agent already recovered from without false-failing correct runs.
- name: Enforce LLM reviewer score threshold (>= 0.7)
if: always()
env:
REVIEWER_THRESHOLD: "0.7"
# Single-quoted heredoc keeps Python literals intact. Threshold
# is read from the env so we don't have to escape dollar signs.
run: |
python - <<'PY'
import json, os, pathlib, sys
threshold = float(os.environ.get('REVIEWER_THRESHOLD', '0.8'))
run_dirs = sorted(pathlib.Path('/tmp/runs').glob('*/'))
if not run_dirs:
print('::warning::No run directory — skipping reviewer threshold check')
sys.exit(0)
run_dir = run_dirs[-1]
task_jsons = sorted(run_dir.rglob('task.json'))
failures = []
missing = []
for p in task_jsons:
d = json.loads(p.read_text(encoding='utf-8'))
lr = d.get('llm_review') or {}
score = lr.get('score')
tid = d.get('task_id', p.parent.name)
if not isinstance(score, (int, float)):
missing.append(tid)
continue
if score < threshold:
issues = (lr.get('issues') or '').splitlines()[0][:120]
failures.append((tid, score, issues))
for tid in missing:
print(f'::warning::{tid}: no llm_review (reviewer skipped or errored)')
if failures:
for tid, score, issues in failures:
print(f'::error::{tid}: llm_review score {score:.2f} < {threshold} — {issues}')
print(f'::error::{len(failures)} task(s) below LLM reviewer threshold {threshold}')
sys.exit(1)
print(f'All {len(task_jsons)} task(s) passed llm_review >= {threshold}')
PY
# Hard gate: fail the job if fewer than PASS_RATE_THRESHOLD of tasks
# passed deterministic criteria. Lets a small number of flakes /
# known-broken tasks through without blocking every PR, while still
# catching systemic regressions. Cancellation (120m timeout) is
# always a fail since a truncated run has no meaningful rate.
- name: Enforce pass-rate threshold (>= 95%)
if: always()
env:
PASS_RATE_THRESHOLD: "0.95"
# The smoke step carries continue-on-error, so its failure only reaches the
# job through this gate. Zero tasks AFTER a failed smoke step means the run
# died before producing anything (bad install, unusable config) — fail closed
# rather than reading an empty set as a clean sweep.
SMOKE_OUTCOME: ${{ steps.smoke.outcome }}
run: |
if [ "${{ steps.smoke.outcome }}" = "cancelled" ]; then
echo "::error::Smoke step cancelled (timeout) — failing without rate check"
exit 1
fi
python - <<'PY'
import json, os, pathlib, sys
threshold = float(os.environ['PASS_RATE_THRESHOLD'])
tasks = sorted(pathlib.Path('/tmp/runs').rglob('task.json'))
results = [json.loads(p.read_text(encoding='utf-8')) for p in tasks]
TENANT_NOT_READY = 'TENANT NOT READY'
def tenant_not_ready(d):
if d.get('final_status') != 'ERROR' or d.get('iteration_count'):
return False
return any(r.get('exit_code') and TENANT_NOT_READY in (r.get('stderr') or '')
for r in d.get('pre_run_results') or [])
infra = [d for d in results if tenant_not_ready(d)]
results = [d for d in results if not tenant_not_ready(d)]
for d in infra:
print(f"::warning::{d.get('task_id')}: excluded from pass rate, tenant not ready: {d.get('error_message')}")
if infra and not results:
print('::error::Every task hit a tenant that is not ready, failing closed')
sys.exit(1)
passed = sum(1 for d in results if d.get('final_status') == 'SUCCESS')
n = len(results)
if n == 0:
if os.environ.get('SMOKE_OUTCOME') == 'failure':
print('::error::Smoke step failed without producing a single task.json '
'— failing closed instead of passing on an empty run')
sys.exit(1)
print('No smoke-tagged tasks matched — skipping rate check')
sys.exit(0)
rate = passed / n
for d in results:
s = d.get('final_status')
if s != 'SUCCESS':
print(f"::warning::{d.get('task_id')}: status={s}")
print(f'Pass rate: {rate:.1%} ({passed}/{n})')
if rate < threshold:
print(f'::error::Pass rate {rate:.1%} < threshold {threshold:.0%}')
sys.exit(1)
PY
# Advisory consistency gate. Does every task YAML in the repo still load
# under the pinned coder_eval's TaskDefinition schema (tests/.coder-eval-version
# — the same engine smoke/ad-hoc/nightly run)? This is the skills-side
# mirror of coder_eval's `validate-skills-yamls` gate: it catches the class
# of bug where a task YAML uses a criterion type/field coder_eval's schema
# doesn't accept, surfacing it here at authoring time instead of weeks later
# (and on the wrong repo) in coder_eval CI.
#
# Validates the WHOLE tree, not just changed files, so pre-existing drift is
# always visible. `continue-on-error` shows the job red for visibility but
# keeps the workflow conclusion green — non-blocking, like the coder_eval
# gate. Flip to a hard gate by removing `continue-on-error` once the tree is
# clean and the convention has bedded in.
validate-task-schema:
name: Validate task schema (advisory)
runs-on: uipath-ubuntu-latest
timeout-minutes: 8
continue-on-error: true
# Skip forks: the private-feed creds for the pinned coder-eval wheel aren't
# available to forked PRs (matches the e2e job's secret requirements).
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# Pin coder_eval to tests/.coder-eval-version so schema validation runs
# against the same engine that actually executes the tasks.
- name: Resolve coder_eval pin
id: ceref
run: echo "version=$(tr -d '[:space:]' < tests/.coder-eval-version)" >> "$GITHUB_OUTPUT"
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
# Host coder-eval CLI = pinned wheel (coder_eval feed + ml-packages for
# deps), constrained to the dependency set frozen into the pinned agent
# image so the schema check resolves the same deps the tasks run under.
# `coder-eval plan` itself validates on the runner -- the image is only
# the constraints source. GH_PACKAGES_TOKEN (read:packages) only; see
# the e2e job's pull step for why never the repo-write PAT.
- name: Install coder-eval
env:
VERSION: ${{ steps.ceref.outputs.version }}
run: |
set -euo pipefail
img="ghcr.io/uipath/coder-eval-agent:$VERSION"
echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
docker pull "$img"
docker run --rm --entrypoint pip "$img" freeze --exclude-editable | grep -vE '^coder-eval| @ ' > "$RUNNER_TEMP/ce-constraints.txt"
uv pip install --system -c "$RUNNER_TEMP/ce-constraints.txt" "coder-eval[litellm]==$VERSION"
- name: Validate every task YAML against the pinned coder_eval schema
# `coder-eval plan` exits non-zero if any file fails to load under the
# current TaskDefinition schema. Job-level continue-on-error keeps a
# failure here non-blocking.
run: |
set -uo pipefail
YAMLS=$(find tests/tasks -type f -name '*.yaml' | grep -v '/_shared/' | sort)
if [ -z "$YAMLS" ]; then
echo "No task YAMLs found under tests/tasks — checkout layout drift?"
exit 1
fi
echo "Validating $(printf '%s\n' "$YAMLS" | wc -l | xargs) task YAML(s) against coder_eval@main."
printf '%s\n' "$YAMLS" | tr '\n' '\0' | xargs -0 coder-eval plan