fix(test): wait for control-plane recovery before retrying deploys

.github/scripts/collect-cluster-diagnostics.sh

@@ -0,0 +1,177 @@
+#!/bin/bash
+
+# ============================================================================
+# Collect Kubernetes cluster diagnostics for functional test post-mortems.
+#
+# Functional tests intermittently fail with `connection reset by peer` or `EOF`
+# against the Radius API. Diagnosing those runs requires distinguishing a
+# control-plane outage (etcd slowness, a kube-apiserver liveness kill) from a
+# Radius bug, and the pod snapshot alone cannot do that. This script captures
+# the missing evidence:
+#
+#   * pod state as YAML, which preserves `lastState.terminated.reason` and
+#     `exitCode` for containers that already restarted (`kubectl describe` has
+#     been observed to omit the `Last State` block entirely)
+#   * cluster-wide events, which record liveness/readiness probe failures and
+#     the resulting kills
+#   * node conditions, which record disk/memory/PID pressure
+#   * the current and previous logs of every kube-system container, including
+#     etcd - the previous instance's log is where an apiserver crash reason and
+#     etcd's `apply request took too long` / `slow fdatasync` warnings live
+#   * verbose /livez and /readyz output, which names the specific failing
+#     apiserver health check
+#
+# Collection is best effort by design: it runs after a test job that may have
+# already started tearing the cluster down, so an individual kubectl failure is
+# recorded in the output and never aborts the rest of the collection.
+# ============================================================================
+
+set -euo pipefail
+
+readonly SCRIPT_NAME="$(basename "$0")"
+
+OUTPUT_DIR=""
+PREFIX="cluster"
+
+usage() {
+    echo "Usage: ${SCRIPT_NAME} --output-dir DIR [--prefix NAME]"
+    echo "Options:"
+    echo "  -o, --output-dir   Directory to write diagnostics into (required)"
+    echo "  -p, --prefix       Filename prefix for the output files"
+    echo "                     (default: cluster)"
+    echo "  -h, --help         Show this help"
+    exit 0
+}
+
+validate_requirements() {
+    if [[ -z "${OUTPUT_DIR}" ]]; then
+        echo "Error: --output-dir is required" >&2
+        exit 1
+    fi
+
+    if ! command -v kubectl >/dev/null; then
+        echo "Error: kubectl is required but not installed" >&2
+        exit 1
+    fi
+}
+
+# capture appends the command line and its combined output to a file. A failing
+# command is recorded rather than propagated so one unavailable resource does
+# not stop the remaining collection.
+capture() {
+    local output_file="$1"
+    shift
+
+    {
+        echo "=============================================================="
+        echo "\$ $*"
+        echo "=============================================================="
+        "$@" 2>&1 || echo "(command exited with code $?)"
+        echo
+    } >>"${output_file}"
+}
+
+collect_pod_state() {
+    # The plain-text snapshot keeps the historical pod-states file intact, and
+    # the YAML dump adds the machine-readable lastState/exitCode fields.
+    capture "${OUTPUT_DIR}/${PREFIX}-pod-states.log" kubectl get pods -A -o wide
+    capture "${OUTPUT_DIR}/${PREFIX}-pod-states.log" kubectl describe pods -A
+
+    # Written raw, with no headers, so the file stays parseable by a YAML
+    # reader. stderr goes to the pod-states log so a failed dump is still
+    # explained there rather than silently producing an empty file.
+    kubectl get pods -A -o yaml >"${OUTPUT_DIR}/${PREFIX}-pods.yaml" \
+        2>>"${OUTPUT_DIR}/${PREFIX}-pod-states.log" || true
+}
+
+collect_events() {
+    capture "${OUTPUT_DIR}/${PREFIX}-events.log" \
+        kubectl get events -A --sort-by=.lastTimestamp
+}
+
+collect_node_state() {
+    capture "${OUTPUT_DIR}/${PREFIX}-nodes.log" kubectl get nodes -o wide
+    capture "${OUTPUT_DIR}/${PREFIX}-nodes.log" kubectl describe nodes
+}
+
+collect_control_plane_health() {
+    local output_file="${OUTPUT_DIR}/${PREFIX}-control-plane-health.log"
+
+    capture "${output_file}" kubectl get --raw "/livez?verbose"
+    capture "${output_file}" kubectl get --raw "/readyz?verbose"
+    capture "${output_file}" kubectl get apiservices -o wide
+}
+
+
+# collect_kube_system_logs writes the current and previous logs of every
+# kube-system container. Timestamps are included so the logs can be correlated
+# with Radius pod logs and with the test output.
+collect_kube_system_logs() {
+    local logs_dir="${OUTPUT_DIR}/${PREFIX}-kube-system-logs"
+    mkdir -p "${logs_dir}"
+
+    local pods
+    if ! pods="$(kubectl get pods -n kube-system \
+        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')"; then
+        echo "Warning: unable to list kube-system pods" >&2
+        return 0
+    fi
+
+    local pod container containers previous_file
+    for pod in ${pods}; do
+        if ! containers="$(kubectl get pod "${pod}" -n kube-system \
+            -o jsonpath='{range .spec.initContainers[*]}{.name}{"\n"}{end}{range .spec.containers[*]}{.name}{"\n"}{end}')"; then
+            continue
+        fi
+
+        for container in ${containers}; do
+            kubectl logs "${pod}" -n kube-system -c "${container}" \
+                --timestamps >"${logs_dir}/${pod}.${container}.log" 2>&1 || true
+
+            # A previous instance only exists once a container has restarted,
+            # and it holds the reason it died. Drop the file when there is no
+            # previous instance so the artifact only contains real restarts.
+            previous_file="${logs_dir}/${pod}.${container}.previous.log"
+            if ! kubectl logs "${pod}" -n kube-system -c "${container}" \
+                --previous --timestamps >"${previous_file}" 2>&1; then
+                rm -f "${previous_file}"
+            fi
+        done
+    done
+}
+
+main() {
+    validate_requirements
+
+    mkdir -p "${OUTPUT_DIR}"
+
+    echo "Collecting cluster diagnostics into ${OUTPUT_DIR}"
+    collect_pod_state
+    collect_events
+    collect_node_state
+    collect_control_plane_health
+    collect_kube_system_logs
+    echo "Cluster diagnostics collection complete"
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -o | --output-dir)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        -p | --prefix)
+            PREFIX="$2"
+            shift 2
+            ;;
+        -h | --help)
+            usage
+            ;;
+        *)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+    esac
+done
+
+main "$@"

.github/workflows/functional-test-cloud.yaml

@@ -64,8 +64,16 @@ env:
   CONTAINER_REGISTRY: ${{ vars.FUNCTIONAL_TEST_CONTAINER_REGISTRY }}
   # Container registry for storing Bicep recipe artifacts
   BICEP_RECIPE_REGISTRY: ${{ vars.FUNCTIONAL_TEST_BICEP_RECIPE_REGISTRY }}
-  # The radius functional test timeout
-  FUNCTIONALTEST_TIMEOUT: 60m
+  # The radius functional test timeout. This bounds `go test` and must stay
+  # comfortably below the job's timeout-minutes: the job also pays for ~10
+  # minutes of setup (KinD cluster, Radius install, Helm charts, cloud logins)
+  # before the tests start, and must still have time afterwards to collect
+  # cluster diagnostics and upload artifacts. If `go test` is allowed to run to
+  # the job timeout, GitHub cancels the job mid-test and the post-test steps -
+  # including diagnostics collection - never run, which is exactly the evidence
+  # needed to debug a hang. The cloud legs normally spend about 5 minutes in
+  # `go test`, so this leaves a wide margin.
+  FUNCTIONALTEST_TIMEOUT: 30m
   # The Azure Location to store test resources
   AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
   # The base directory for storing test logs
@@ -981,6 +989,12 @@ jobs:
           # Restore AWS Bicep types
           bicep restore ./test/functional-portable/corerp/cloud/resources/testdata/aws-logs-loggroup.bicep --force
       - name: Run functional tests
+        # A step-level timeout, unlike the job-level one, still lets the
+        # `if: always()` steps below run - so a hung suite is followed by
+        # diagnostics collection rather than a cancelled job. Kept above
+        # FUNCTIONALTEST_TIMEOUT so `go test` times out first and prints its
+        # goroutine dump. Mirrors the pattern in long-running-azure.yaml.
+        timeout-minutes: 35
         run: |
           set -euo pipefail
 
@@ -1049,23 +1063,20 @@ jobs:
           result_directory: dist/functional_test/
           comment_mode: failures
 
-      - name: Collect Pod details
+      - name: Collect cluster diagnostics
         if: always()
-        # Diagnostic-only snapshot of cluster-wide pod state for post-mortem. A
-        # `kubectl describe pods -A` can race pod teardown and return a transient
-        # NotFound, so never let it fail the job.
+        # Diagnostic-only snapshot for post-mortem: pod state (including the
+        # previous container instance's termination reason), events, node
+        # conditions, control-plane health, and kube-system container logs.
+        # These commands can race cluster teardown and return a transient
+        # NotFound, so never let them fail the job.
         continue-on-error: true
         env:
           MATRIX_NAME: ${{ matrix.name }}
         run: |
-          POD_STATE_LOG_FILENAME="${RADIUS_CONTAINER_LOG_BASE}/${MATRIX_NAME}-tests-pod-states.log"
-          mkdir -p "$(dirname "${POD_STATE_LOG_FILENAME}")"
-          {
-            echo "kubectl get pods -A"
-            kubectl get pods -A
-            echo "kubectl describe pods -A"
-            kubectl describe pods -A
-          } >> "${POD_STATE_LOG_FILENAME}"
+          ./.github/scripts/collect-cluster-diagnostics.sh \
+            --output-dir "${RADIUS_CONTAINER_LOG_BASE}" \
+            --prefix "${MATRIX_NAME}-tests"
 
       - name: Upload container logs
         if: always()

.github/workflows/functional-test-noncloud.yaml

@@ -710,23 +710,20 @@ jobs:
           retention-days: 30
           if-no-files-found: error
 
-      - name: Collect Pod details
+      - name: Collect cluster diagnostics
         if: always()
-        # Diagnostic-only snapshot of cluster-wide pod state for post-mortem. A
-        # `kubectl describe pods -A` can race pod teardown and return a transient
-        # NotFound, so never let it fail the job.
+        # Diagnostic-only snapshot for post-mortem: pod state (including the
+        # previous container instance's termination reason), events, node
+        # conditions, control-plane health, and kube-system container logs.
+        # These commands can race cluster teardown and return a transient
+        # NotFound, so never let them fail the job.
         continue-on-error: true
         env:
           MATRIX_NAME: ${{ matrix.name }}
         run: |
-          POD_STATE_LOG_FILENAME="${RADIUS_CONTAINER_LOG_BASE}/${MATRIX_NAME}-tests-pod-states.log"
-          mkdir -p "$(dirname "$POD_STATE_LOG_FILENAME")"
-          {
-            echo "kubectl get pods -A"
-            kubectl get pods -A
-            echo "kubectl describe pods -A"
-            kubectl describe pods -A
-          }  >> "${POD_STATE_LOG_FILENAME}"
+          ./.github/scripts/collect-cluster-diagnostics.sh \
+            --output-dir "${RADIUS_CONTAINER_LOG_BASE}" \
+            --prefix "${MATRIX_NAME}-tests"
 
       - name: Upload container logs
         if: always()

.github/workflows/long-running-azure.yaml

@@ -451,21 +451,18 @@ jobs:
           make test-functional-all-cloud
           make test-functional-all-noncloud
 
-      - name: Collect Pod details
+      - name: Collect cluster diagnostics
         if: always()
-        # Diagnostic-only snapshot of cluster-wide pod state for post-mortem. A
-        # `kubectl describe pods -A` can race pod teardown and return a transient
-        # NotFound, so never let it fail the job.
+        # Diagnostic-only snapshot for post-mortem: pod state (including the
+        # previous container instance's termination reason), events, node
+        # conditions, control-plane health, and kube-system container logs.
+        # These commands can race cluster teardown and return a transient
+        # NotFound, so never let them fail the job.
         continue-on-error: true
         run: |
-          POD_STATE_LOG_FILENAME="${RADIUS_CONTAINER_LOG_BASE}/all-tests-pod-states.log"
-          mkdir -p $(dirname $POD_STATE_LOG_FILENAME)
-          {
-            echo "kubectl get pods -A"
-            kubectl get pods -A
-            echo "kubectl describe pods -A"
-            kubectl describe pods -A
-          } >> "${POD_STATE_LOG_FILENAME}"
+          ./.github/scripts/collect-cluster-diagnostics.sh \
+            --output-dir "${RADIUS_CONTAINER_LOG_BASE}" \
+            --prefix "all-tests"
 
       - name: Upload container logs
         if: always()

test/functional-portable/corerp/cloud/resources/aci_test.go

@@ -40,12 +40,13 @@ import (
 func Test_ACI(t *testing.T) {
 	// Disabled: this test dominates the corerp-cloud functional leg's wall-clock
 	// time. It provisions real Azure Container Instances (plus a VNet/NSG/ILB) and
-	// is wrapped in a 2x retry with 60s backoff to tolerate the subscription-shared
-	// 'StandardCores' ACI quota (ContainerGroupQuotaReached). When the quota is
-	// exhausted by concurrent CI runs the deploy is retried end-to-end, so a single
-	// run can take ~11-12 minutes (vs <1 minute for every other test in the leg) and
-	// roughly doubles the cloud workflow's total time. Re-enable once the ACI tests
-	// are isolated onto their own quota-aware lane. See #12044 and #12163.
+	// is wrapped in a 3 minute retry budget with 60s backoff to tolerate the
+	// subscription-shared 'StandardCores' ACI quota (ContainerGroupQuotaReached).
+	// When the quota is exhausted by concurrent CI runs the deploy is retried
+	// end-to-end, so a single run can take ~11-12 minutes (vs <1 minute for every
+	// other test in the leg) and roughly doubles the cloud workflow's total time.
+	// Re-enable once the ACI tests are isolated onto their own quota-aware lane.
+	// See #12044 and #12163.
 	t.Skip("Test_ACI is temporarily disabled: real ACI provisioning + quota retries dominate corerp-cloud CI time. See #12044, #12163.")
 
 	name := "aci-app"
@@ -56,7 +57,7 @@ func Test_ACI(t *testing.T) {
 
 	test := rp.NewRPTest(t, name, []rp.TestStep{
 		{
-			Executor:             step.NewDeployExecutor(template).WithRetry(2, 60*time.Second, isTransientAzureError),
+			Executor:             step.NewDeployExecutor(template).WithRetry(3*time.Minute, 60*time.Second, isTransientAzureError),
 			SkipObjectValidation: true,
 			RPResources: &validation.RPResourceSet{
 				Resources: []validation.RPResource{

test/functional-portable/corerp/cloud/resources/radiuscore_azure_mysql_portallink_test.go

@@ -127,7 +127,7 @@ func Test_RadiusCore_AzureMySql_PortalLink(t *testing.T) {
 				"azureSubscriptionId="+azureSubscriptionID,
 				"azureResourceGroupName="+azureResourceGroupName,
 				"password=not-prod-password",
-			).WithRetry(2, 60*time.Second, isTransientAzureError),
+			).WithRetry(3*time.Minute, 60*time.Second, isTransientAzureError),
 			RPResources: &validation.RPResourceSet{
 				Resources: []validation.RPResource{
 					{Name: recipePackName, Type: validation.CoreRecipePacksResource},