Skip to content

feat(uipath-agents): HTTP Request built-in tool for inline agents in a Flow #14325

feat(uipath-agents): HTTP Request built-in tool for inline agents in a Flow

feat(uipath-agents): HTTP Request built-in tool for inline agents in a Flow #14325

name: Smoke RPA Skill Tests (Windows)
concurrency:
group: smoke-rpa-skills-${{ github.head_ref || github.ref }}
# Only `synchronize` moves the head SHA; `opened` and `reopened` can fire on a
# commit a run is already in flight for, and a cancelled run reports the
# required context as `cancelled`. See docs/REQUIRED-CHECKS.md Rule 3a.
cancel-in-progress: ${{ github.event.action == 'synchronize' }}
on:
# No `paths:` filter on purpose. This workflow is a required status
# check; if `paths:` excluded a PR the workflow wouldn't run, no
# check would be reported, and merge would block forever. Instead we
# always trigger and let the `detect` job short-circuit via
# `skip=true` — the gated job is then reported as skipped, which
# GitHub counts as a pass for required checks.
pull_request:
workflow_dispatch:
# Exempt the first-party `coder-eval` wheel from safe-chain's
# minimum-package-age suppression so a same-day pin in
# tests/.coder-eval-version still resolves — see smoke-skills.yml for the full
# rationale. Workflow-level so any job added later inherits it.
env:
SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: coder-eval,anthropic
jobs:
detect:
runs-on: uipath-ubuntu-latest
name: Detect changed RPA skills
outputs:
task_globs: ${{ steps.detect.outputs.task_globs }}
skip: ${{ steps.detect.outputs.skip }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0
- name: Detect changed Windows-tagged skills and map to test tasks
id: detect
env:
DISABLE_SMOKE: ${{ vars.DISABLE_SMOKE }}
run: |
# Kill switch: repo variable DISABLE_SMOKE=true skips the smoke tests
# (e.g. while coder-eval runs fail on expired registry tokens rather
# than on skill regressions). Emitting skip=true keeps this required
# check (and the gated one) green. Unset or any other value: run.
if [ "$DISABLE_SMOKE" = "true" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "Smoke tests disabled (vars.DISABLE_SMOKE == 'true') — skipping"
exit 0
fi
BASE_REF="${{ github.base_ref }}"
if [ -z "$BASE_REF" ]; then BASE_REF="main"; fi
CHANGED=$(git diff --name-only origin/$BASE_REF...HEAD)
# Discover the set of Windows-tagged task files (tag = `windows` in the
# `tags:` line). Source of truth for "what runs on this Windows
# workflow" — adding/removing a `windows` tag is the opt-in/out.
WINDOWS_TASKS=$(grep -rEl '^tags:.*\bwindows\b' tests/tasks/ 2>/dev/null | sort -u)
if [ -z "$WINDOWS_TASKS" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "No tasks tagged 'windows' found — skipping"
exit 0
fi
# Skills that own at least one windows-tagged task. Used to map
# `skills/<X>/` changes to the right test set.
WINDOWS_SKILLS=$(echo "$WINDOWS_TASKS" | sed -nE 's|^tests/tasks/([^/]+)/.*|\1|p' | sort -u)
# Globs are relative to the `tests/` working dir of the smoke job.
ALL_GLOBS=$(echo "$WINDOWS_TASKS" | sed 's|^tests/||' | tr '\n' ' ' | xargs)
# Infra change → run every windows-tagged task.
if echo "$CHANGED" | grep -qE '^tests/(experiments|_shared)/|^tests/[^/]+\.(py|yaml|toml)$|^\.github/workflows/smoke-rpa-skills\.yml$'; then
echo "task_globs=$ALL_GLOBS" >> "$GITHUB_OUTPUT"
echo "Running all Windows smoke tasks (test infrastructure changed)"
exit 0
fi
# Skill-scoped change → run only that skill's windows-tagged tasks.
GLOBS=""
for skill in $WINDOWS_SKILLS; do
if echo "$CHANGED" | grep -qE "^(skills/${skill}/|tests/tasks/${skill}/)"; then
for t in $WINDOWS_TASKS; do
case "$t" in
"tests/tasks/${skill}/"*) GLOBS="$GLOBS ${t#tests/}";;
esac
done
fi
done
GLOBS=$(echo "$GLOBS" | xargs)
if [ -z "$GLOBS" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
echo "No Windows-tagged skill changes detected — skipping"
else
echo "task_globs=$GLOBS" >> "$GITHUB_OUTPUT"
echo "Will test: $GLOBS"
fi
# The Windows smoke job installs coder-eval on the host (tempdir driver,
# no Docker), so it can't read the agent image itself. Freeze the pinned
# image's deps here on Linux and hand them over as an artifact.
- name: Freeze coder-eval-agent deps as install constraints
if: steps.detect.outputs.skip != 'true'
run: |
set -euo pipefail
img="ghcr.io/uipath/coder-eval-agent:$(tr -d '[:space:]' < tests/.coder-eval-version)"
echo "${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
docker pull "$img"
docker run --rm --entrypoint pip "$img" freeze --exclude-editable | grep -vE '^coder-eval| @ ' > "$RUNNER_TEMP/ce-constraints.txt"
docker logout ghcr.io
- name: Upload coder-eval constraints
if: steps.detect.outputs.skip != 'true'
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: ce-constraints
path: ${{ runner.temp }}/ce-constraints.txt
retention-days: 1
smoke:
needs: detect
if: needs.detect.outputs.skip != 'true'
runs-on: uipath-windows-latest
# RPA smoke tasks spin up Helm on each `uip rpa` call (30–60s each). A
# typical 3-task smoke run with cold Helm + three tasks runs ~25–40
# min; bump the ceiling so we don't cancel mid-task.
timeout-minutes: 90
name: Run RPA skill smoke tests (Windows)
# Disable uip CLI version-sync: tasks run in tempdir (no container) and
# share one host ~/.uipath / global uip, so a `uip login status` re-pin
# downgrades the CLI for later tasks. Mirrors daily.sh.
env:
UIPATH_CLI_DISABLE_VERSION_SYNC: "1"
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# Pin coder_eval to tests/.coder-eval-version so RPA smoke runs the same
# engine as the skills smoke, ad-hoc, and nightly paths. shell: bash
# because windows-latest defaults to PowerShell (no `tr`).
- name: Resolve coder_eval pin
id: ceref
shell: bash
run: echo "version=$(tr -d '[:space:]' < tests/.coder-eval-version)" >> "$GITHUB_OUTPUT"
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '20'
- uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
with:
dotnet-version: '8.0.x'
# Host coder-eval CLI = pinned wheel (coder_eval feed + ml-packages for
# deps). Windows RPA tasks run under the tempdir driver, so no agent image.
#
# claude-agent-sdk releases sometimes ship without a win_amd64 wheel
# (0.2.144, 0.2.157, 0.2.160). uv then builds the sdist, which bundles no
# claude.exe, and every task dies at agent_crash ("Claude Code not
# found"). --only-binary makes the resolver skip any release lacking a
# wheel for this platform and take the newest one that has it — no
# per-release denylist, no upper cap. The check below fails fast if a
# wheel ever arrives without the bundled binary.
#
# Constrained to the pinned agent image's frozen deps (frozen by the
# detect job). claude-agent-sdk is dropped from them so --only-binary
# keeps picking a release that ships a win_amd64 wheel.
- name: Download coder-eval constraints
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: ce-constraints
path: ${{ runner.temp }}
- name: Install coder-eval
shell: bash
env:
VERSION: ${{ steps.ceref.outputs.version }}
run: |
set -euo pipefail
grep -v '^claude-agent-sdk' "$RUNNER_TEMP/ce-constraints.txt" > "$RUNNER_TEMP/ce-constraints-win.txt"
uv pip install --system --only-binary claude-agent-sdk \
-c "$RUNNER_TEMP/ce-constraints-win.txt" "coder-eval==$VERSION"
python3 -c "import pathlib, sys, claude_agent_sdk as s; exe = pathlib.Path(s.__file__).parent / '_bundled' / 'claude.exe'; print(f'claude-agent-sdk {s.__version__}: {exe}'); sys.exit(0 if exe.is_file() else f'::error::claude-agent-sdk {s.__version__} has no bundled claude.exe')"
# The feed allows anonymous reads, so it is added without credentials.
- name: Configure NuGet feed for Helm packages
shell: bash
run: |
dotnet nuget add source \
"https://uipath.pkgs.visualstudio.com/Public.Feeds/_packaging/UiPath-Internal/nuget/v3/index.json" \
--name UiPath-Internal
# Install from GitHub Packages `@dev` — the live prerelease train
# published on every push to `cli/main`. Public npm mirrors the same
# releases ~2 weeks later, so pulling `@latest` from public npm made
# RPA-Windows tasks exercise a stale CLI that could miss freshly
# renamed/added verbs. `@dev` keeps this runner in lockstep with the
# Linux smoke image, which builds from the same `@dev` tag (see
# smoke-skills.yml and tests/docker/Dockerfile).
#
# The `@uipath` scope is mapped to npm.pkg.github.com via a temp npmrc.
# Token is GH_PACKAGES_TOKEN (read:packages only), never the repo-write
# PAT: this workflow is `pull_request`-triggered, so anyone who can push
# a branch to this public repo can read the secret it is handed.
#
# EVERY package is pinned to the explicit `@dev` dist-tag: the same feed
# also carries a FROZEN `alpha` tag (1.197.0-alpha.*) and the public
# `latest` line — never rely on `latest`/unspecified here, which could
# resolve to a stale line. The npmrc is written and deleted inside this
# step so the token never lingers on the runner.
#
# Tools are installed with raw `npm install -g @uipath/<tool>@dev`, NOT
# `uip tools install`: on a `-dev` CLI the latter resolves the release
# channel to `stable` and fails to find `-dev` tool builds (cli PR #2650).
# uipcli's ToolManager then discovers them on disk under
# `<npm-prefix>/@uipath/`, so `uip rpa …` / `uip rpa-legacy …` resolve
# without triggering auto-install. Job-level UIPATH_CLI_DISABLE_VERSION_SYNC
# keeps a `-dev` CLI from trying to re-align tools through the (stable-only)
# channel path at runtime.
#
# `npm install -g` runs from a fresh tempdir so any future workspace-aware
# `package.json` at the repo root cannot pull the install into a workspace.
- name: Install uip CLI + RPA tools (GitHub Packages @dev)
shell: bash
env:
GH_PACKAGES_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GH_PAT }}
run: |
set -euo pipefail
if [ -z "${GH_PACKAGES_TOKEN:-}" ]; then
echo "::error::GH_PACKAGES_TOKEN is empty — cannot authenticate to the GitHub Packages @dev feed"
exit 1
fi
install_dir="$(mktemp -d)"
cd "$install_dir"
echo "Installing from $install_dir (workspace-isolated)"
npmrc="$install_dir/.npmrc"
{
echo "@uipath:registry=https://npm.pkg.github.com/"
echo "//npm.pkg.github.com/:_authToken=${GH_PACKAGES_TOKEN}"
} > "$npmrc"
export NPM_CONFIG_USERCONFIG="$npmrc"
npm install -g @uipath/cli@dev
uip --version
for tool in @uipath/rpa-tool @uipath/rpa-legacy-tool; do
echo "Installing ${tool}@dev"
if ! npm install -g "${tool}@dev"; then
echo "::error::${tool}@dev failed to install — no build on the dev train?"
exit 1
fi
done
rm -f "$npmrc"
unset NPM_CONFIG_USERCONFIG
echo "--- Installed @uipath/* packages ---"
npm ls -g --depth=0 2>/dev/null | grep '@uipath/' || true
echo "--- uip tools list ---"
uip tools list --output json
# Pre-auth uip as a real licensed Studio user via Studio's own e2e
# auth helper (vendored from Studio/.ci/helm-e2e/oauth-login.mjs).
# Why this and not client_credentials:
# * Helm's HelmFeatureGate runs TWO checks: sign-in state AND
# license SKU entitlement (HelmLicenseSkuFeatureSourceService).
# A licensed user account carries the Studio SKU and clears
# both gates; an External App / client_credentials principal
# typically doesn't.
# * Studio Desktop scopes (StudioWebBackend, OrchestratorApiUserAccess,
# LLMGateway, ...) are user-only — not requestable via
# client_credentials grant at all.
# * Studio engineers themselves use this exact script to drive
# Helm e2e; we get free upkeep when login UI changes.
# The script drives a headless Puppeteer browser through the
# auth-code+PKCE flow, exchanges the code for tokens, then writes
# the standard ~/.uipath/.auth file the JS CLI reads on disk
# (refresh token included — much longer effective lifetime than
# env-auth's ~1h).
- name: Install Puppeteer (auth helper dep)
shell: bash
run: npm install --no-save puppeteer
- name: Authenticate uip as licensed Studio user
id: auth
shell: bash
env:
AUTHORITY: https://alpha.uipath.com
EMAIL: ${{ secrets.UIPATH_EMAIL }}
PASSWORD: ${{ secrets.UIPATH_PASSWORD }}
TENANT: ${{ vars.UIPATH_TENANT }}
ORG: ${{ vars.UIPATH_ORG }}
# Drop screenshots + DOM dumps here so we can upload them as an
# artifact for post-mortem on auth failures.
AUTH_DEBUG_DIR: ${{ github.workspace }}/auth-debug
run: |
set -euo pipefail
mkdir -p "$AUTH_DEBUG_DIR"
# Auth via puppeteer is occasionally flaky on alpha (org-select
# SPA renders late, or the login form inputs intermittently lose
# focus). Retry the login flow up to 3 times before giving up.
for attempt in 1 2 3; do
echo "--- Auth attempt $attempt ---"
if node .github/scripts/uipath-oauth-login.mjs; then
break
fi
if [ "$attempt" = "3" ]; then
echo "::error::Auth failed after 3 attempts"
exit 1
fi
sleep 5
done
# Smoke check the auth context is recognized. Failure here fails
# the job — we'd rather not run smoke tasks against a broken
# login state and chase mysterious sign-in errors per task.
uip login status --output json
# Always upload the auth-debug screenshots / dumps so we have a
# post-mortem artifact even on success (baseline for comparison).
- name: Upload auth-debug artifacts
if: always() && steps.auth.outcome != 'skipped'
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: auth-debug-${{ github.run_id }}
path: ${{ github.workspace }}/auth-debug
if-no-files-found: ignore
retention-days: 7
- name: Pre-warm Helm (download NuGet package before tests)
shell: bash
run: |
mkdir -p /tmp/helm-warmup && cd /tmp/helm-warmup
uip rpa list-instances --output json 2>&1 || true
taskkill //F //IM UiPath.Studio.Helm.exe 2>/dev/null || true
# Model guard, deliberately its OWN step (mirror of smoke-skills.yml): the smoke
# step below carries continue-on-error, so a guard inside it is swallowed and the
# failure surfaces only as a downstream "no task.json" error. Fail here instead,
# named, before the run starts.
- name: Require an agent model
shell: bash
env:
AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
run: ': "${AGENT_MODEL:?unset — set the CLAUDE_CODE_MODEL repo variable}"'
- name: Run RPA smoke tests
env:
SKILLS_REPO_PATH: ${{ github.workspace }}
# Route the agent through AWS Bedrock. The Anthropic workspace
# hit its API usage limit (resets 2026-05-01), so DirectRoute
# is blocked. Bedrock has a separate quota.
API_BACKEND: bedrock
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
AWS_REGION: ${{ vars.AWS_REGION }}
BEDROCK_MODEL: ${{ vars.BEDROCK_MODEL }}
# The gate's agent model, passed on the command line rather than pinned in
# experiments/smoke-windows.yaml (see smoke-skills.yml for why). No literal
# fallback: an unset variable fails the guard step above rather than gating a
# model nobody configured.
AGENT_MODEL: ${{ vars.CLAUDE_CODE_MODEL }}
working-directory: tests
id: smoke
shell: bash
run: |
# Run each task separately, killing Helm between tasks so one
# task's stale Studio state doesn't leak into the next. Track
# per-task exit codes so one failure doesn't abort the loop
# but still propagates to the overall step outcome.
shopt -s globstar nullglob
overall_exit=0
for task in ${{ needs.detect.outputs.task_globs }}; do
# TODO(rpa legacy): Bedrock deprecated Opus 4.6 mid-2026-04
# and Opus 4.1 IDs return 400 'invalid model identifier'
# on the EU regional inference profiles we have access to.
# Sonnet trips Bedrock's content filter on the full legacy
# XAML baseline reliably (3/3 retries). Skip the task in
# CI until we have a working Bedrock model for it; the
# task YAML itself is still tracked for local runs and
# for an eventual model fix.
case "$task" in
tasks/uipath-rpa/legacy/*)
echo "::warning::Skipping $task — pending Bedrock Opus model fix"
continue
;;
esac
echo "--- Killing leftover Helm/Studio processes ---"
taskkill //F //IM UiPath.Studio.Helm.exe 2>/dev/null || true
echo "--- Running: $task ---"
# We previously overrode the legacy task to Opus on Bedrock
# to dodge a content filter that tripped on the full legacy
# XAML baseline. Opus 4.6 was deprecated on Bedrock, and
# both 4-1 (un-versioned) and the suggested
# claude-opus-4-1-20250805-v1:0 also returned 400 'invalid
# model identifier' on the EU regional inference profiles
# we have access to. Drop the override and rely on the
# content-filter retry below to absorb intermittent blocks.
for attempt in 1 2 3; do
echo "--- $task attempt $attempt ---"
if coder-eval run "$task" \
-e experiments/smoke-windows.yaml --model "$AGENT_MODEL" --tags smoke -j 1 -v; then
break
fi
# Only retry on Bedrock content-filter ERRORs (not on real
# task failures — those are real signal). Inspect the most
# recent task.json and bail out if final_status is anything
# other than ERROR with content-filter details.
latest_json="$(ls -t runs/*/default/*/00/task.json 2>/dev/null | head -n 1 || true)"
if [ -n "$latest_json" ] && LATEST_JSON="$latest_json" python3 -c "import json,os,sys; d=json.load(open(os.environ['LATEST_JSON'])); det=d.get('error_details') or {}; msg=json.dumps(det)+(d.get('error_message') or ''); sys.exit(0 if d.get('final_status')=='ERROR' and 'content filter' in msg.lower() else 2)"; then
echo "::warning::$task: content-filter error on attempt $attempt — retrying"
if [ "$attempt" = "3" ]; then
overall_exit=1
fi
continue
fi
overall_exit=1
break
done
done
exit $overall_exit
continue-on-error: true
# Upload as early as possible so the report artifact is available
# for post-mortem regardless of whether later gate/summary steps
# crash.
- name: Upload HTML / JSON eval report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: eval-report-${{ github.run_id }}
path: |
tests/runs/*/experiment.html
tests/runs/*/experiment.md
tests/runs/*/experiment.json
tests/runs/*/experiment.log
tests/runs/*/*/variant.html
tests/runs/*/*/variant.md
tests/runs/*/*/variant.json
tests/runs/**/task.html
tests/runs/**/task.json
tests/runs/**/task.log
if-no-files-found: warn
retention-days: 14
- name: Summarize results
if: always()
working-directory: tests
shell: bash
run: |
run_dir=$(ls -td runs/*/ 2>/dev/null | head -n 1)
if [ -z "$run_dir" ]; then
echo "::warning::No run directory found — nothing to summarize"
exit 0
fi
echo "Run directory: $run_dir"
echo ""
echo "## Task results"
python - <<'PY'
import json, pathlib
# Skip runs/latest — it's a symlink/copy of the most recent
# timestamped run dir, so including it double-counts that task.
run_dirs = [d for d in sorted(pathlib.Path('runs').glob('*/'))
if d.name != 'latest']
rows = []
for run_dir in run_dirs:
for p in sorted(run_dir.rglob('task.json')):
d = json.loads(p.read_text(encoding='utf-8'))
lr = d.get('llm_review') or {}
rows.append((
d.get('task_id', '?'),
d.get('final_status', '?'),
d.get('weighted_score'),
lr.get('score'),
))
if not rows:
print(' (no task.json files found)')
width = max((len(r[0]) for r in rows), default=4)
for task_id, status, score, llm_score in rows:
score_s = f"{score:.2f}" if isinstance(score, (int, float)) else ' —'
llm_s = f"{llm_score:.2f}" if isinstance(llm_score, (int, float)) else ' —'
print(f" {task_id:<{width}} status={status:<8} score={score_s} llm={llm_s}")
PY
echo ""
echo "HTML report artifact: eval-report-${{ github.run_id }}"
echo "Download under the workflow run's Artifacts section and open experiment.html."
# Single global gate: each task must hit weighted_score >= 0.7.
# Replaces both the per-criterion `pass_threshold` gate and the
# LLM-reviewer threshold gate — `weighted_score` is the
# deterministic aggregate of the YAML success_criteria, so one
# uniform bar at 0.7 is enough to enforce quality without
# re-tuning each criterion's threshold by hand.
- name: Enforce per-task score threshold (>= 0.7)
if: always()
working-directory: tests
shell: bash
env:
TASK_THRESHOLD: "0.7"
run: |
python - <<'PY'
import json, os, pathlib, sys
threshold = float(os.environ.get('TASK_THRESHOLD', '0.7'))
# Skip runs/latest — it's a symlink/copy of the most recent
# timestamped run dir, so including it double-counts that task.
run_dirs = [d for d in sorted(pathlib.Path('runs').glob('*/'))
if d.name != 'latest']
task_jsons = []
for run_dir in run_dirs:
task_jsons.extend(sorted(run_dir.rglob('task.json')))
if not task_jsons:
print('::error::No task.json files found — nothing to gate')
sys.exit(1)
failures, missing = [], []
for p in task_jsons:
d = json.loads(p.read_text(encoding='utf-8'))
tid = d.get('task_id', p.parent.name)
score = d.get('weighted_score')
if not isinstance(score, (int, float)):
missing.append(tid)
continue
if score < threshold:
failures.append((tid, score))
for tid in missing:
print(f'::error::{tid}: missing weighted_score')
for tid, score in failures:
print(f'::error::{tid}: weighted_score {score:.2f} < {threshold}')
fails = len(failures) + len(missing)
total = len(task_jsons)
print(f'Results: {total - fails}/{total} task(s) >= {threshold}')
if fails:
sys.exit(1)
PY
- name: Fail if smoke step was cancelled
if: steps.smoke.outcome == 'cancelled'
shell: bash
run: exit 1