diff --git a/.github/workflows/check_deployments.yml b/.github/workflows/check_deployments.yml index 22ba6fabf4..9b9b074013 100644 --- a/.github/workflows/check_deployments.yml +++ b/.github/workflows/check_deployments.yml @@ -32,7 +32,9 @@ jobs: deploy: name: Deploy missing workflows needs: check - if: github.ref == 'refs/heads/main' && needs.check.outputs.has-missing == 'true' + # always() is required: without it this job is skipped whenever the check + # job fails, which is precisely when there is something to deploy. + if: always() && github.ref == 'refs/heads/main' && needs.check.outputs.has-missing == 'true' runs-on: ubuntu-latest strategy: matrix: @@ -58,10 +60,25 @@ jobs: credentials: 'https://${{ env.GITHUB_USER }}:${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }}@github.com/' - name: Deploy missing workflows run: | + failed=() while IFS= read -r repo_dir; do [ -z "$repo_dir" ] && continue - echo "Deploying $repo_dir ..." - planemo workflow_upload --namespace iwc-workflows "$repo_dir" - done <<< "${{ needs.check.outputs.missing-repos }}" + echo "::group::Deploying $repo_dir" + if planemo workflow_upload --namespace iwc-workflows "$repo_dir"; then + echo "Deployed $repo_dir" + else + echo "::error::Failed to deploy $repo_dir" + failed+=("$repo_dir") + fi + echo "::endgroup::" + done <<< "$MISSING_REPOS" + if [ ${#failed[@]} -ne 0 ]; then + echo "Failed to deploy ${#failed[@]} workflow(s):" + printf ' %s\n' "${failed[@]}" + exit 1 + fi env: + # Passed via the environment rather than interpolated into the script, + # so that repository paths cannot be expanded as shell syntax. + MISSING_REPOS: ${{ needs.check.outputs.missing-repos }} GITHUB_TOKEN: ${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }} diff --git a/scripts/check_missing_deployments.py b/scripts/check_missing_deployments.py index 5b4c26227c..e4b9edff61 100644 --- a/scripts/check_missing_deployments.py +++ b/scripts/check_missing_deployments.py @@ -3,16 +3,22 @@ iwc-workflows GitHub organization, and optionally write the list of repository directories that need redeployment.""" +import argparse import json import os import subprocess import sys +import time from pathlib import Path import yaml WORKFLOWS_DIR = Path(__file__).resolve().parent.parent / "workflows" ORG = "iwc-workflows" +# Transient GitHub API errors (secondary rate limits in particular) are common +# when querying ~100 repositories in a row, so retry before giving up. +API_RETRIES = 3 +API_RETRY_DELAY = 10 def get_expected_release(repo_dir): @@ -30,12 +36,27 @@ def get_expected_release(repo_dir): def release_exists(repo_name, tag): - """Check if a release tag exists in the iwc-workflows org via gh CLI.""" - result = subprocess.run( - ["gh", "api", f"repos/{ORG}/{repo_name}/releases/tags/{tag}"], - capture_output=True, - ) - return result.returncode == 0 + """Check if a release tag exists in the iwc-workflows org via gh CLI. + + Only a genuine 404 means the release is missing. Any other failure (rate + limiting, network trouble, an expired token) is retried and then raised, + so that an API problem is never mistaken for a missing deployment. + """ + for attempt in range(1, API_RETRIES + 1): + result = subprocess.run( + ["gh", "api", "--silent", f"repos/{ORG}/{repo_name}/releases/tags/{tag}"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return True + stderr = result.stderr or "" + if "HTTP 404" in stderr or "Not Found" in stderr: + return False + if attempt < API_RETRIES: + print(f" API error for {repo_name} (attempt {attempt}), retrying: {stderr.strip()}") + time.sleep(API_RETRY_DELAY) + raise RuntimeError(f"Could not determine release status of {ORG}/{repo_name} {tag}: {stderr.strip()}") def find_all_repos(): @@ -47,6 +68,14 @@ def find_all_repos(): def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fail-on-missing", + action="store_true", + help="Exit non-zero when deployments are missing (the default outside GitHub Actions)", + ) + args = parser.parse_args() + repos = find_all_repos() missing = [] @@ -80,7 +109,12 @@ def main(): f.write(f"missing-repos<