ci: add isolated DSW SWE-bench release pipeline

.github/scripts/dsw-swe-verified/dispatch-release-benchmark.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${RELEASE_TAG:?RELEASE_TAG is required}"
+: "${RELEASE_ID:?RELEASE_ID is required}"
+: "${QWEN_REF:?QWEN_REF is required}"
+: "${QWEN_COMMIT:?QWEN_COMMIT is required}"
+: "${INSTANCE_LIMIT:?INSTANCE_LIMIT is required}"
+: "${BENCHMARK_IDEMPOTENCY_KEY:?BENCHMARK_IDEMPOTENCY_KEY is required}"
+: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
+
+script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+pool_root="${DSW_POOL_ROOT:-/mnt/workspace/qwen-benchmark-pool}"
+pool_bin="${POOL_BIN:-${pool_root}/venv/bin/qwen-benchmark-pool}"
+python_bin="${POOL_PYTHON:-${pool_root}/venv/bin/python}"
+dataset_root="${SWE_VERIFIED_DATASET_ROOT:-${pool_root}/datasets/swe-bench-verified}"
+database_url="${BENCHMARK_POOL_DATABASE_URL:-postgresql://qwen_benchmark@127.0.0.1:55432/qwen_benchmark_dsw_release_v1}"
+model_name="${OPENAI_MODEL:-qwen3.7-max}"
+output_root="${GITHUB_WORKSPACE:-$(pwd)}/benchmark-output"
+
+if [[ ! "${INSTANCE_LIMIT}" =~ ^[0-9]+$ ]] || (( INSTANCE_LIMIT < 1 || INSTANCE_LIMIT > 500 )); then
+  echo "INSTANCE_LIMIT must be between 1 and 500" >&2
+  exit 2
+fi
+for required_path in "${pool_bin}" "${python_bin}" "${dataset_root}"; do
+  if [[ ! -e "${required_path}" ]]; then
+    echo "Required DSW resource is missing: ${required_path}" >&2
+    exit 2
+  fi
+done
+
+mkdir -p "${output_root}"
+manifest_path="${output_root}/manifest.json"
+manifest_args=(
+  --dataset-root "${dataset_root}"
+  --limit "${INSTANCE_LIMIT}"
+  --output "${manifest_path}"
+)
+if [[ -n "${BENCHMARK_INSTANCE_ID:-}" ]]; then
+  manifest_args+=(--instance-id "${BENCHMARK_INSTANCE_ID}")
+fi
+"${python_bin}" "${script_root}/make-manifest.py" "${manifest_args[@]}"
+
+export BENCHMARK_POOL_DATABASE_URL="${database_url}"
+"${pool_bin}" init-db >/dev/null
+submit_json="$(
+  "${pool_bin}" submit \
+    --idempotency-key "${BENCHMARK_IDEMPOTENCY_KEY}" \
+    --suite "dsw_release_swe_verified_v1" \
+    --dataset "swe-bench/swe-bench-verified" \
+    --dataset-revision "2" \
+    --task-prefix "swe-bench/" \
+    --qwen-ref "${QWEN_REF}" \
+    --qwen-commit "${QWEN_COMMIT}" \
+    --model "${model_name}" \
+    --manifest "${manifest_path}" \
+    --max-attempts 2 \
+    --infra-failure-threshold 0 \
+    --repository "${GITHUB_REPOSITORY}" \
+    --release-id "${RELEASE_ID}" \
+    --release-tag "${RELEASE_TAG}" \
+    --github-run-url "${GITHUB_RUN_URL:-}"
+)"
+run_id="$("${python_bin}" -c 'import json,sys; print(json.load(sys.stdin)["run_id"])' <<< "${submit_json}")"
+
+jq -n \
+  --arg status "QUEUED" \
+  --arg run_id "${run_id}" \
+  --arg release_tag "${RELEASE_TAG}" \
+  --arg qwen_ref "${QWEN_REF}" \
+  --arg qwen_commit "${QWEN_COMMIT}" \
+  --argjson expected_instances "${INSTANCE_LIMIT}" \
+  --argjson executor_count "${EXECUTOR_COUNT:-10}" \
+  '{
+    status: $status,
+    run_id: $run_id,
+    release_tag: $release_tag,
+    qwen_ref: $qwen_ref,
+    qwen_commit: $qwen_commit,
+    expected_instances: $expected_instances,
+    executor_count: $executor_count
+  }' > "${output_root}/dispatch-receipt.json"
+
+if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
+  {
+    echo "run_id=${run_id}"
+    echo "status=QUEUED"
+  } >> "${GITHUB_OUTPUT}"
+fi
+
+echo "Queued ${INSTANCE_LIMIT} SWE-bench Verified instances as ${run_id}."

.github/scripts/dsw-swe-verified/make-manifest.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset-root", type=Path, required=True)
+    parser.add_argument("--limit", type=int, required=True)
+    parser.add_argument("--instance-id")
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+
+    instance_ids = sorted(
+        path.name
+        for path in args.dataset_root.iterdir()
+        if path.is_dir() and "__" in path.name
+    )
+    if len(instance_ids) != 500:
+        raise SystemExit(
+            f"Expected exactly 500 SWE-bench Verified instances, found {len(instance_ids)}"
+        )
+    if args.instance_id:
+        if args.limit != 1:
+            raise SystemExit("--instance-id requires --limit 1")
+        if args.instance_id not in instance_ids:
+            raise SystemExit(f"Unknown SWE-bench Verified instance: {args.instance_id}")
+        selected = [args.instance_id]
+    else:
+        selected = instance_ids[: args.limit]
+    if len(selected) != args.limit:
+        raise SystemExit(f"Requested {args.limit} instances, found {len(selected)}")
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(
+        json.dumps(
+            {
+                "dataset": "swe-bench/swe-bench-verified",
+                "dataset_revision": "2",
+                "expected_instances": len(selected),
+                "instance_ids": selected,
+            },
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/dsw-swe-verified-release.yml

@@ -0,0 +1,122 @@
+name: DSW SWE-bench Verified Release
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Existing test/prerelease to receive the result
+        required: true
+        type: string
+      qwen_release_tag:
+        description: Published Qwen npm release to evaluate (defaults to release_tag)
+        required: false
+        type: string
+      instance_limit:
+        description: Number of Verified instances (500 is the full suite)
+        required: true
+        default: '1'
+        type: choice
+        options:
+          - '1'
+          - '5'
+          - '500'
+      executor_count:
+        description: Concurrent DSW executors
+        required: true
+        default: '10'
+        type: choice
+        options:
+          - '1'
+          - '4'
+          - '10'
+      instance_id:
+        description: Exact Verified instance when instance_limit is 1
+        required: false
+        default: sympy__sympy-20590
+        type: string
+
+permissions:
+  contents: write
+
+concurrency:
+  group: dsw-swe-verified-release
+  cancel-in-progress: false
+
+jobs:
+  benchmark:
+    name: Dispatch SWE-bench Verified to DSW
+    runs-on: [self-hosted, Linux, X64, qwen-benchmark-dsw]
+    timeout-minutes: 15
+    env:
+      RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }}
+      QWEN_REF: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.qwen_release_tag || inputs.release_tag }}
+      INSTANCE_LIMIT: ${{ github.event_name == 'release' && '500' || inputs.instance_limit }}
+      EXECUTOR_COUNT: ${{ github.event_name == 'release' && '10' || inputs.executor_count }}
+      BENCHMARK_INSTANCE_ID: ${{ github.event_name == 'release' && '' || inputs.instance_id }}
+      RELEASE_ID: ${{ github.event_name == 'release' && github.event.release.id || '' }}
+      BENCHMARK_TRIGGER: ${{ github.event_name }}
+      BENCHMARK_IDEMPOTENCY_KEY: dsw-release-v1-${{ github.run_id }}
+      GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+    steps:
+      - name: Checkout pipeline
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          fetch-depth: 1
+
+      - name: Resolve release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          curl --fail --silent --show-error --location \
+            --header "Accept: application/vnd.github+json" \
+            --header "Authorization: Bearer ${GH_TOKEN}" \
+            --header "X-GitHub-Api-Version: 2022-11-28" \
+            "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" \
+            > release.json
+          test "$(jq -r .draft release.json)" = "false"
+          qwen_ref="${QWEN_REF}"
+          if [[ "$(jq -r .prerelease release.json)" == "true" ]]; then
+            override="$(
+              jq -r '.body // ""' release.json \
+                | sed -nE 's/^Benchmark-Qwen-Ref:[[:space:]]*(v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?)[[:space:]]*$/\1/p' \
+                | head -n 1
+            )"
+            if [[ -n "${override}" ]]; then
+              qwen_ref="${override}"
+            fi
+          fi
+          git fetch --depth=1 --force origin "refs/tags/${qwen_ref}:refs/tags/${qwen_ref}"
+          release_commit="$(git rev-list -n 1 "${qwen_ref}^{commit}")"
+          release_id="$(jq -r .id release.json)"
+          {
+            echo "qwen_ref=${qwen_ref}"
+            echo "release_commit=${release_commit}"
+            echo "release_id=${release_id}"
+            echo "release_url=$(jq -r .html_url release.json)"
+          } >> "${GITHUB_OUTPUT}"
+
+      - name: Dispatch to persistent DSW pool
+        id: dispatch
+        env:
+          QWEN_REF: ${{ steps.release.outputs.qwen_ref }}
+          QWEN_COMMIT: ${{ steps.release.outputs.release_commit }}
+          RELEASE_ID: ${{ steps.release.outputs.release_id }}
+        run: |
+          .github/scripts/dsw-swe-verified/dispatch-release-benchmark.sh
+
+      - name: Record dispatch receipt
+        run: |
+          {
+            echo "### DSW SWE-bench Verified dispatch"
+            echo
+            echo "- Pool run: \`${{ steps.dispatch.outputs.run_id }}\`"
+            echo "- State: queued"
+            echo "- Cases: ${INSTANCE_LIMIT}"
+            echo "- DSW executors: ${EXECUTOR_COUNT}"
+            echo "- The persistent DSW publisher will update the Release after the run reaches a validated terminal state."
+          } >> "${GITHUB_STEP_SUMMARY}"

docs/design/dsw-swe-verified-release-pipeline.md

@@ -0,0 +1,61 @@
+# DSW SWE-bench Verified Release Pipeline
+
+This pipeline is an isolated implementation of:
+
+`GitHub Release -> short DSW dispatch job -> persistent 10-executor pool -> DSW publisher -> Release result`
+
+It does not use or modify the workflow, service, state, or result markers from
+PR #7584.
+
+## Production behavior
+
+- A published Release starts the workflow from the Release tag's target commit.
+- The Release tag is resolved to its immutable Git commit.
+- The full 500-instance SWE-bench Verified manifest is frozen before dispatch.
+- The Action writes the manifest and Release callback metadata to PostgreSQL,
+  records the pool `run_id`, and ends without waiting for the benchmark.
+- A persistent Coordinator and ten persistent executors process the run. Each
+  executor atomically claims one task and runs one Harbor/Docker trial at a time.
+- Harbor live trial directories stay on local NVMe. Completed attempt artifacts
+  are copied to OSS without depending on OSS POSIX permission operations.
+- The Coordinator maintains leases, heartbeat recovery, one infrastructure
+  retry, run counters, a circuit breaker, and the completion gate.
+- A persistent DSW publisher watches terminal runs and actively updates the
+  triggering Release.
+- A score is written to the Release only when all 500 instances have a unique
+  terminal state and the run status is `SUCCEEDED`.
+- A `QUARANTINED` or pipeline-error run writes status and counts, but never a
+  score.
+
+## Isolation boundaries
+
+- Runner label: `qwen-benchmark-dsw`
+- Workflow: `.github/workflows/dsw-swe-verified-release.yml`
+- Suite: `dsw_release_swe_verified_v1`
+- PostgreSQL database: `qwen_benchmark_dsw_release_v1`
+- Runtime: `/mnt/workspace/qwen-benchmark-dsw-release-v1`
+- Model credential: `/mnt/workspace/qwen-benchmark-dsw-release-v1/config/model.key`
+  (`root:github-runner`, mode `0640`)
+- OSS: `/mnt/data/qwen-benchmark/dsw-release-v1`
+- Release markers: `qwen-code-dsw-swe-verified`
+
+Docker image layers may use the DSW host cache, but experiment state and
+artifacts do not share paths or tables with another benchmark pipeline.
+
+## Branch validation
+
+Publish an isolated prerelease whose target commit is on this branch. GitHub
+then evaluates this branch's `release.published` workflow and automatically
+starts the DSW job with `instance_limit=500` and `executor_count=10`; no manual
+DSW dispatch is involved.
+
+For an event-driven test prerelease, a single body line such as
+`Benchmark-Qwen-Ref: v0.20.0-nightly.20260722.b98306b7e` selects an existing
+published Qwen npm version while keeping the result on the isolated POC Release.
+This override is accepted only for prereleases. A normal Release always evaluates
+its own tag.
+
+`workflow_dispatch` remains available for explicit diagnostics and reruns.
+Manual validation defaults to one instance to bound time and model cost. Both
+triggers are asynchronous: Actions records a dispatch receipt but does not stay
+alive for the benchmark duration.