feat: publish benchmark results to GitHub

.github/workflows/benchmark-dispatch.yml

@@ -1,6 +1,8 @@
 name: 'Dispatch SWE-bench Verified Benchmark'
 
 on:
+  release:
+    types: [published]
   workflow_dispatch:
     inputs:
       qwen_ref:
@@ -20,37 +22,62 @@ on:
 jobs:
   dispatch:
     name: 'Submit benchmark run'
+    if: '${{ github.event_name != ''release'' || github.event.release.prerelease == false }}'
     runs-on: [self-hosted, Linux, X64, qwen-benchmark]
     timeout-minutes: 10
     permissions:
       contents: 'read'
 
     steps:
-      - name: 'Checkout benchmarked release'
-        uses: 'actions/checkout@v4'
-        with:
-          ref: '${{ github.event.release.tag_name || inputs.qwen_ref }}'
-          fetch-depth: 1
-
       - name: 'Resolve immutable release commit'
         id: 'revision'
         shell: 'bash'
-        run: 'echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"'
+        env:
+          QWEN_REF: '${{ github.event.release.tag_name || inputs.qwen_ref }}'
+          GH_TOKEN: '${{ github.token }}'
+        run: |-
+          set -euo pipefail
+          object="$(curl --fail --silent --show-error \
+            --header "Authorization: Bearer ${GH_TOKEN}" \
+            --header 'Accept: application/vnd.github+json' \
+            --header 'X-GitHub-Api-Version: 2022-11-28' \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/ref/tags/${QWEN_REF}")"
+          object_type="$(jq -r '.object.type' <<<"${object}")"
+          object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          while [[ "${object_type}" == 'tag' ]]; do
+            object="$(curl --fail --silent --show-error \
+              --header "Authorization: Bearer ${GH_TOKEN}" \
+              --header 'Accept: application/vnd.github+json' \
+              --header 'X-GitHub-Api-Version: 2022-11-28' \
+              "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/tags/${object_sha}")"
+            object_type="$(jq -r '.object.type' <<<"${object}")"
+            object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          done
+          [[ "${object_type}" == 'commit' && "${object_sha}" =~ ^[0-9a-f]{40}$ ]]
+          echo "commit=${object_sha}" >> "${GITHUB_OUTPUT}"
 
-      - name: 'Submit benchmark request to local ECS service'
+      - name: 'Submit benchmark request to local ECS queue'
         env:
           QWEN_REF: '${{ github.event.release.tag_name || inputs.qwen_ref }}'
           QWEN_COMMIT: '${{ steps.revision.outputs.commit }}'
-          SUITE: '${{ inputs.suite }}'
+          SUITE: "${{ inputs.suite || 'swebench_verified_harbor_smoke' }}"
+          RELEASE_ID: '${{ github.event.release.id }}'
+          TRIGGER: '${{ github.event_name }}'
         run: |-
           set -euo pipefail
-          response="$(sudo -n /usr/local/sbin/qwen-benchmark-dispatch \
-            --repository "${GITHUB_REPOSITORY}" \
-            --qwen-ref "${QWEN_REF}" \
-            --qwen-commit "${QWEN_COMMIT}" \
-            --suite "${SUITE}" \
-            --github-run-id "${GITHUB_RUN_ID}" \
-            --github-run-attempt "${GITHUB_RUN_ATTEMPT}")"
+          args=(
+            --repository "${GITHUB_REPOSITORY}"
+            --qwen-ref "${QWEN_REF}"
+            --qwen-commit "${QWEN_COMMIT}"
+            --suite "${SUITE}"
+            --trigger "${TRIGGER}"
+            --github-run-id "${GITHUB_RUN_ID}"
+            --github-run-attempt "${GITHUB_RUN_ATTEMPT}"
+          )
+          if [[ -n "${RELEASE_ID}" ]]; then
+            args+=(--release-id "${RELEASE_ID}")
+          fi
+          response="$(sudo -n /usr/local/sbin/qwen-benchmark-dispatch "${args[@]}")"
           echo "${response}" | jq .
           echo "### Benchmark queued" >> "$GITHUB_STEP_SUMMARY"
           echo "${response}" | jq -r '"- Run ID: `" + .run_id + "`\n- Status: `" + .status + "`"' >> "$GITHUB_STEP_SUMMARY"

.github/workflows/stale.yml

@@ -26,6 +26,10 @@ on:
         options:
           - 'swebench_verified_gold_smoke'
           - 'swebench_verified_harbor_smoke'
+      release_id:
+        description: 'Optional GitHub Release ID to update after the benchmark.'
+        required: false
+        type: string
 
 jobs:
   stale:
@@ -75,31 +79,55 @@ jobs:
       contents: read
 
     steps:
-      - name: 'Checkout benchmarked release'
-        uses: 'actions/checkout@v4'
-        with:
-          ref: '${{ inputs.qwen_ref }}'
-          fetch-depth: 1
-
       - name: 'Resolve immutable release commit'
         id: revision
         shell: bash
-        run: 'echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"'
+        env:
+          QWEN_REF: '${{ inputs.qwen_ref }}'
+          GH_TOKEN: '${{ github.token }}'
+        run: |-
+          set -euo pipefail
+          object="$(curl --fail --silent --show-error \
+            --header "Authorization: Bearer ${GH_TOKEN}" \
+            --header 'Accept: application/vnd.github+json' \
+            --header 'X-GitHub-Api-Version: 2022-11-28' \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/ref/tags/${QWEN_REF}")"
+          object_type="$(jq -r '.object.type' <<<"${object}")"
+          object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          while [[ "${object_type}" == 'tag' ]]; do
+            object="$(curl --fail --silent --show-error \
+              --header "Authorization: Bearer ${GH_TOKEN}" \
+              --header 'Accept: application/vnd.github+json' \
+              --header 'X-GitHub-Api-Version: 2022-11-28' \
+              "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/git/tags/${object_sha}")"
+            object_type="$(jq -r '.object.type' <<<"${object}")"
+            object_sha="$(jq -r '.object.sha' <<<"${object}")"
+          done
+          [[ "${object_type}" == 'commit' && "${object_sha}" =~ ^[0-9a-f]{40}$ ]]
+          echo "commit=${object_sha}" >> "${GITHUB_OUTPUT}"
 
-      - name: 'Queue benchmark on local ECS service'
+      - name: 'Queue benchmark in local ECS worker'
         env:
           QWEN_REF: '${{ inputs.qwen_ref }}'
           QWEN_COMMIT: '${{ steps.revision.outputs.commit }}'
           SUITE: '${{ inputs.suite }}'
+          RELEASE_ID: '${{ inputs.release_id }}'
+          TRIGGER: workflow_dispatch
         run: |-
           set -euo pipefail
-          response="$(sudo -n /usr/local/sbin/qwen-benchmark-dispatch \
-            --repository "${GITHUB_REPOSITORY}" \
-            --qwen-ref "${QWEN_REF}" \
-            --qwen-commit "${QWEN_COMMIT}" \
-            --suite "${SUITE}" \
-            --github-run-id "${GITHUB_RUN_ID}" \
-            --github-run-attempt "${GITHUB_RUN_ATTEMPT}")"
+          args=(
+            --repository "${GITHUB_REPOSITORY}"
+            --qwen-ref "${QWEN_REF}"
+            --qwen-commit "${QWEN_COMMIT}"
+            --suite "${SUITE}"
+            --trigger "${TRIGGER}"
+            --github-run-id "${GITHUB_RUN_ID}"
+            --github-run-attempt "${GITHUB_RUN_ATTEMPT}"
+          )
+          if [[ -n "${RELEASE_ID}" ]]; then
+            args+=(--release-id "${RELEASE_ID}")
+          fi
+          response="$(sudo -n /usr/local/sbin/qwen-benchmark-dispatch "${args[@]}")"
           echo "${response}" | jq .
           echo "### Benchmark queued" >> "$GITHUB_STEP_SUMMARY"
           echo "${response}" | jq -r '"- Run ID: `" + .run_id + "`\n- Status: `" + .status + "`"' >> "$GITHUB_STEP_SUMMARY"