Fix check_deployments never deploying missing workflows

.github/workflows/check_deployments.yml

@@ -32,7 +32,9 @@ jobs:
   deploy:
     name: Deploy missing workflows
     needs: check
-    if: github.ref == 'refs/heads/main' && needs.check.outputs.has-missing == 'true'
+    # always() is required: without it this job is skipped whenever the check
+    # job fails, which is precisely when there is something to deploy.
+    if: always() && github.ref == 'refs/heads/main' && needs.check.outputs.has-missing == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -58,10 +60,25 @@ jobs:
         credentials: 'https://${{ env.GITHUB_USER }}:${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }}@github.com/'
     - name: Deploy missing workflows
       run: |
+        failed=()
         while IFS= read -r repo_dir; do
           [ -z "$repo_dir" ] && continue
-          echo "Deploying $repo_dir ..."
-          planemo workflow_upload --namespace iwc-workflows "$repo_dir"
-        done <<< "${{ needs.check.outputs.missing-repos }}"
+          echo "::group::Deploying $repo_dir"
+          if planemo workflow_upload --namespace iwc-workflows "$repo_dir"; then
+            echo "Deployed $repo_dir"
+          else
+            echo "::error::Failed to deploy $repo_dir"
+            failed+=("$repo_dir")
+          fi
+          echo "::endgroup::"
+        done <<< "$MISSING_REPOS"
+        if [ ${#failed[@]} -ne 0 ]; then
+          echo "Failed to deploy ${#failed[@]} workflow(s):"
+          printf '  %s\n' "${failed[@]}"
+          exit 1
+        fi
       env:
+        # Passed via the environment rather than interpolated into the script,
+        # so that repository paths cannot be expanded as shell syntax.
+        MISSING_REPOS: ${{ needs.check.outputs.missing-repos }}
         GITHUB_TOKEN: ${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }}

scripts/check_missing_deployments.py

@@ -3,16 +3,22 @@
 iwc-workflows GitHub organization, and optionally write the list of
 repository directories that need redeployment."""
 
+import argparse
 import json
 import os
 import subprocess
 import sys
+import time
 from pathlib import Path
 
 import yaml
 
 WORKFLOWS_DIR = Path(__file__).resolve().parent.parent / "workflows"
 ORG = "iwc-workflows"
+# Transient GitHub API errors (secondary rate limits in particular) are common
+# when querying ~100 repositories in a row, so retry before giving up.
+API_RETRIES = 3
+API_RETRY_DELAY = 10
 
 
 def get_expected_release(repo_dir):
@@ -30,12 +36,27 @@ def get_expected_release(repo_dir):
 
 
 def release_exists(repo_name, tag):
-    """Check if a release tag exists in the iwc-workflows org via gh CLI."""
-    result = subprocess.run(
-        ["gh", "api", f"repos/{ORG}/{repo_name}/releases/tags/{tag}"],
-        capture_output=True,
-    )
-    return result.returncode == 0
+    """Check if a release tag exists in the iwc-workflows org via gh CLI.
+
+    Only a genuine 404 means the release is missing. Any other failure (rate
+    limiting, network trouble, an expired token) is retried and then raised,
+    so that an API problem is never mistaken for a missing deployment.
+    """
+    for attempt in range(1, API_RETRIES + 1):
+        result = subprocess.run(
+            ["gh", "api", "--silent", f"repos/{ORG}/{repo_name}/releases/tags/{tag}"],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode == 0:
+            return True
+        stderr = result.stderr or ""
+        if "HTTP 404" in stderr or "Not Found" in stderr:
+            return False
+        if attempt < API_RETRIES:
+            print(f"      API error for {repo_name} (attempt {attempt}), retrying: {stderr.strip()}")
+            time.sleep(API_RETRY_DELAY)
+    raise RuntimeError(f"Could not determine release status of {ORG}/{repo_name} {tag}: {stderr.strip()}")
 
 
 def find_all_repos():
@@ -47,6 +68,14 @@ def find_all_repos():
 
 
 def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--fail-on-missing",
+        action="store_true",
+        help="Exit non-zero when deployments are missing (the default outside GitHub Actions)",
+    )
+    args = parser.parse_args()
+
     repos = find_all_repos()
     missing = []
 
@@ -80,7 +109,12 @@ def main():
             f.write(f"missing-repos<<EOF\n{repo_list}\nEOF\n")
             f.write(f"has-missing={'true' if missing else 'false'}\n")
 
-    return 1 if missing else 0
+    # Inside Actions, missing deployments are reported through the has-missing
+    # output so that the deploy job can act on them. Failing here instead would
+    # mark the run red for the very condition the workflow exists to fix.
+    if missing and (args.fail_on_missing or not github_output):
+        return 1
+    return 0
 
 
 if __name__ == "__main__":