feat: publish benchmark results to GitHub

.github/workflows/benchmark-dispatch.yml

@@ -1,6 +1,8 @@
 name: 'Dispatch SWE-bench Verified Benchmark'
 
 on:
+  release:
+    types: [published]
   workflow_dispatch:
     inputs:
       qwen_ref:
@@ -20,35 +22,56 @@ on:
 jobs:
   dispatch:
     name: 'Submit benchmark run'
+    if: '${{ github.event_name != ''release'' || github.event.release.prerelease == false }}'
     runs-on: [self-hosted, Linux, X64, qwen-benchmark]
     timeout-minutes: 10
     permissions:
       contents: 'read'
 
     steps:
-      - name: 'Checkout benchmarked release'
-        uses: 'actions/checkout@v4'
-        with:
-          ref: '${{ github.event.release.tag_name || inputs.qwen_ref }}'
-          fetch-depth: 1
-
       - name: 'Resolve immutable release commit'
         id: 'revision'
         shell: 'bash'
-        run: 'echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"'
+        env:
+          QWEN_REF: '${{ github.event.release.tag_name || inputs.qwen_ref }}'
+          GH_TOKEN: '${{ github.token }}'
+        run: |-
+          set -euo pipefail
+          object="$(curl --fail --silent --show-error \
+            --header "Authorization: Bearer ${GH_TOKEN}" \
+            --header 'Accept: application/vnd.github+json' \
+            --header 'X-GitHub-Api-Version: 2022-11-28' \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/ref/tags/${QWEN_REF}")"
+          object_type="$(jq -r '.object.type' <<<"${object}")"
+          object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          while [[ "${object_type}" == 'tag' ]]; do
+            object="$(curl --fail --silent --show-error \
+              --header "Authorization: Bearer ${GH_TOKEN}" \
+              --header 'Accept: application/vnd.github+json' \
+              --header 'X-GitHub-Api-Version: 2022-11-28' \
+              "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/tags/${object_sha}")"
+            object_type="$(jq -r '.object.type' <<<"${object}")"
+            object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          done
+          [[ "${object_type}" == 'commit' && "${object_sha}" =~ ^[0-9a-f]{40}$ ]]
+          echo "commit=${object_sha}" >> "${GITHUB_OUTPUT}"
 
       - name: 'Submit benchmark request to local ECS service'
         env:
           QWEN_REF: '${{ github.event.release.tag_name || inputs.qwen_ref }}'
           QWEN_COMMIT: '${{ steps.revision.outputs.commit }}'
-          SUITE: '${{ inputs.suite }}'
+          SUITE: "${{ inputs.suite || 'swebench_verified_harbor_smoke' }}"
+          RELEASE_ID: '${{ github.event.release.id }}'
+          TRIGGER: '${{ github.event_name }}'
         run: |-
           set -euo pipefail
           response="$(sudo -n /usr/local/sbin/qwen-benchmark-dispatch \
             --repository "${GITHUB_REPOSITORY}" \
             --qwen-ref "${QWEN_REF}" \
             --qwen-commit "${QWEN_COMMIT}" \
             --suite "${SUITE}" \
+            --release-id "${RELEASE_ID}" \
+            --trigger "${TRIGGER}" \
             --github-run-id "${GITHUB_RUN_ID}" \
             --github-run-attempt "${GITHUB_RUN_ATTEMPT}")"
           echo "${response}" | jq .

.github/workflows/stale.yml

@@ -26,6 +26,10 @@ on:
         options:
           - 'swebench_verified_gold_smoke'
           - 'swebench_verified_harbor_smoke'
+      release_id:
+        description: 'Optional GitHub Release ID to update after the benchmark.'
+        required: false
+        type: string
 
 jobs:
   stale:
@@ -75,29 +79,49 @@ jobs:
       contents: read
 
     steps:
-      - name: 'Checkout benchmarked release'
-        uses: 'actions/checkout@v4'
-        with:
-          ref: '${{ inputs.qwen_ref }}'
-          fetch-depth: 1
-
       - name: 'Resolve immutable release commit'
         id: revision
         shell: bash
-        run: 'echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"'
+        env:
+          QWEN_REF: '${{ inputs.qwen_ref }}'
+          GH_TOKEN: '${{ github.token }}'
+        run: |-
+          set -euo pipefail
+          object="$(curl --fail --silent --show-error \
+            --header "Authorization: Bearer ${GH_TOKEN}" \
+            --header 'Accept: application/vnd.github+json' \
+            --header 'X-GitHub-Api-Version: 2022-11-28' \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/ref/tags/${QWEN_REF}")"
+          object_type="$(jq -r '.object.type' <<<"${object}")"
+          object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          while [[ "${object_type}" == 'tag' ]]; do
+            object="$(curl --fail --silent --show-error \
+              --header "Authorization: Bearer ${GH_TOKEN}" \
+              --header 'Accept: application/vnd.github+json' \
+              --header 'X-GitHub-Api-Version: 2022-11-28' \
+              "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/tags/${object_sha}")"
+            object_type="$(jq -r '.object.type' <<<"${object}")"
+            object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          done
+          [[ "${object_type}" == 'commit' && "${object_sha}" =~ ^[0-9a-f]{40}$ ]]
+          echo "commit=${object_sha}" >> "${GITHUB_OUTPUT}"
 
       - name: 'Queue benchmark on local ECS service'
         env:
           QWEN_REF: '${{ inputs.qwen_ref }}'
           QWEN_COMMIT: '${{ steps.revision.outputs.commit }}'
           SUITE: '${{ inputs.suite }}'
+          RELEASE_ID: '${{ inputs.release_id }}'
+          TRIGGER: workflow_dispatch
         run: |-
           set -euo pipefail
           response="$(sudo -n /usr/local/sbin/qwen-benchmark-dispatch \
             --repository "${GITHUB_REPOSITORY}" \
             --qwen-ref "${QWEN_REF}" \
             --qwen-commit "${QWEN_COMMIT}" \
             --suite "${SUITE}" \
+            --release-id "${RELEASE_ID}" \
+            --trigger "${TRIGGER}" \
             --github-run-id "${GITHUB_RUN_ID}" \
             --github-run-attempt "${GITHUB_RUN_ATTEMPT}")"
           echo "${response}" | jq .

benchmark-service/deploy/qwen-benchmark-dispatch

@@ -9,12 +9,14 @@ repository=''
 qwen_ref=''
 qwen_commit=''
 suite=''
+release_id=''
+trigger='workflow_dispatch'
 github_run_id=''
 github_run_attempt=''
 
 while (($#)); do
   case "$1" in
-    --repository|--qwen-ref|--qwen-commit|--suite|--github-run-id|--github-run-attempt)
+    --repository|--qwen-ref|--qwen-commit|--suite|--release-id|--trigger|--github-run-id|--github-run-attempt)
       key="${1#--}"
       shift
       (($#)) || { echo "missing value for --${key}" >&2; exit 2; }
@@ -23,6 +25,8 @@ while (($#)); do
         qwen-ref) qwen_ref="$1" ;;
         qwen-commit) qwen_commit="$1" ;;
         suite) suite="$1" ;;
+        release-id) release_id="$1" ;;
+        trigger) trigger="$1" ;;
         github-run-id) github_run_id="$1" ;;
         github-run-attempt) github_run_attempt="$1" ;;
       esac
@@ -36,6 +40,8 @@ done
 [[ "$qwen_ref" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || { echo 'qwen-ref must be a release version' >&2; exit 2; }
 [[ "$qwen_commit" =~ ^[0-9a-f]{40}$ ]] || { echo 'qwen-commit must be a 40-character SHA' >&2; exit 2; }
 [[ "$suite" =~ ^swebench_verified_(gold_smoke|qwen_smoke|harbor_smoke)$ ]] || { echo 'suite is not allowed' >&2; exit 2; }
+[[ -z "$release_id" || "$release_id" =~ ^[0-9]+$ ]] || { echo 'release-id must be numeric' >&2; exit 2; }
+[[ "$trigger" == 'release' || "$trigger" == 'workflow_dispatch' ]] || { echo 'trigger is not allowed' >&2; exit 2; }
 [[ "$github_run_id" =~ ^[0-9]+$ && "$github_run_attempt" =~ ^[0-9]+$ ]] || { echo 'invalid GitHub run metadata' >&2; exit 2; }
 
 set -a
@@ -48,9 +54,11 @@ payload="$(jq -n \
   --arg qwen_ref "$qwen_ref" \
   --arg qwen_commit "$qwen_commit" \
   --arg suite "$suite" \
+  --arg release_id "$release_id" \
+  --arg trigger "$trigger" \
   --arg github_run_id "$github_run_id" \
   --arg github_run_attempt "$github_run_attempt" \
-  '{repository: $repository, qwen_ref: $qwen_ref, qwen_commit: $qwen_commit, suite: $suite, trigger: "workflow_dispatch", github_run_id: ($github_run_id | tonumber), github_run_attempt: ($github_run_attempt | tonumber)}')"
+  '{repository: $repository, qwen_ref: $qwen_ref, qwen_commit: $qwen_commit, suite: $suite, trigger: $trigger, release_id: (if $release_id == "" then null else ($release_id | tonumber) end), github_run_id: ($github_run_id | tonumber), github_run_attempt: ($github_run_attempt | tonumber)}')"
 
 curl --fail-with-body --silent --show-error \
   -X POST "$API_URL" \

benchmark-service/qwen_benchmark/publisher.py

@@ -20,7 +20,8 @@ def publish_check(
         "Authorization": f"Bearer {settings.github_token}",
         "X-GitHub-Api-Version": "2022-11-28",
     }
-    conclusion = "success" if run["status"] == "SUCCEEDED" else "failure"
+    succeeded = run["status"] == "SUCCEEDED"
+    conclusion = "success" if succeeded else "failure"
     release_id = json.loads(run["request_json"]).get("release_id")
     if release_id:
         _update_release_body(settings, run, summary, headers, int(release_id))
@@ -33,12 +34,8 @@ def publish_check(
             "status": "completed",
             "conclusion": conclusion,
             "output": {
-                "title": f"{summary['resolved_instances']} / {summary['expected_instances']} resolved",
-                "summary": (
-                    f"Run `{run['run_id']}` completed with "
-                    f"{summary['resolved_instances']} resolved instances and "
-                    f"{summary['error_instances']} infrastructure errors."
-                ),
+                "title": _check_title(summary, succeeded),
+                "summary": _check_summary(summary, succeeded),
             },
         },
         timeout=30,
@@ -60,22 +57,91 @@ def _update_release_body(
     response.raise_for_status()
     existing_body = response.json().get("body") or ""
     marker = "<!-- qwen-code-benchmark -->"
-    section = "\n".join(
-        [
-            marker,
-            "## Qwen Code benchmark",
-            "",
-            f"- Suite: `{summary['suite']}`",
-            f"- Qwen Code version: `{summary['qwen_version']}`",
-            f"- Commit: `{summary['qwen_commit']}`",
-            f"- Result: **{summary['resolved_instances']} / {summary['expected_instances']} resolved**",
-            f"- Run ID: `{summary['run_id']}`",
-            "",
-        ]
-    )
+    section = _release_section(marker, summary, run["status"] == "SUCCEEDED")
     if marker in existing_body:
         body = existing_body.split(marker, 1)[0].rstrip() + "\n\n" + section
     else:
         body = existing_body.rstrip() + "\n\n" + section
     patch = httpx.patch(url, headers=headers, json={"body": body}, timeout=30)
     patch.raise_for_status()
+
+
+def _check_title(summary: dict[str, Any], succeeded: bool) -> str:
+    if not succeeded:
+        return "Benchmark failed (not scored)"
+    return (
+        f"{_score(summary):.2f}% — "
+        f"{summary['resolved_instances']} / {summary['expected_instances']} resolved"
+    )
+
+
+def _check_summary(summary: dict[str, Any], succeeded: bool) -> str:
+    if not succeeded:
+        return (
+            f"Dataset: `{summary['dataset']}` at `{summary['dataset_revision']}`.\n\n"
+            f"Execution: {summary['completed_instances']} / "
+            f"{summary['expected_instances']} cases completed.\n\n"
+            f"Run `{summary['run_id']}` failed before a valid score was available."
+        )
+    return (
+        f"Dataset: `{summary['dataset']}` at `{summary['dataset_revision']}`.\n\n"
+        f"Execution: {summary['completed_instances']} / "
+        f"{summary['expected_instances']} cases completed; "
+        f"{summary['resolved_instances']} resolved, "
+        f"{summary['unresolved_instances']} unresolved, and "
+        f"{summary['error_instances']} infrastructure errors.\n\n"
+        f"Score: **{_score(summary):.2f}%**. Run `{summary['run_id']}`."
+    )
+
+
+def _release_section(marker: str, summary: dict[str, Any], succeeded: bool) -> str:
+    version = summary["qwen_version"] or summary["qwen_ref"]
+    commit = summary["qwen_commit"]
+    commit_url = f"https://github.com/{summary['repository']}/commit/{commit}"
+    lines = [
+        marker,
+        "## Qwen Code benchmark",
+        "",
+        "| Field | Result |",
+        "| --- | --- |",
+        f"| Status | **{'Completed' if succeeded else 'Failed — not scored'}** |",
+        f"| Dataset | `{summary['dataset']}` at `{summary['dataset_revision']}` |",
+        f"| Suite | `{summary['suite']}` |",
+        f"| Evaluation | {_evaluation_method(summary['runner_mode'])} |",
+        f"| Cases | {summary['completed_instances']} / {summary['expected_instances']} completed |",
+    ]
+    if succeeded:
+        lines.extend(
+            [
+                "| Results | "
+                f"{summary['resolved_instances']} resolved · "
+                f"{summary['unresolved_instances']} unresolved · "
+                f"{summary['error_instances']} infrastructure errors |",
+                f"| Score | **{_score(summary):.2f}%** |",
+            ]
+        )
+    else:
+        lines.append("| Score | Not published |")
+    lines.extend(
+        [
+            f"| Qwen Code | `{version}` · [`{commit[:7]}`]({commit_url}) |",
+            f"| Run | `{summary['run_id']}` |",
+            "",
+        ]
+    )
+    return "\n".join(lines)
+
+
+def _score(summary: dict[str, Any]) -> float:
+    expected = int(summary["expected_instances"])
+    if expected <= 0:
+        return 0.0
+    return int(summary["resolved_instances"]) * 100.0 / expected
+
+
+def _evaluation_method(runner_mode: str) -> str:
+    return {
+        "harbor": "Qwen Code agent (Harbor)",
+        "qwen": "Qwen Code agent (SWE-bench harness)",
+        "gold": "Gold patch harness validation",
+    }.get(runner_mode, runner_mode)