fix(test): wait for control-plane recovery before retrying deploys

.github/scripts/collect-cluster-diagnostics.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+
+# ============================================================================
+# Collect Kubernetes cluster diagnostics for functional test post-mortems.
+#
+# Functional tests intermittently fail with `connection reset by peer` or `EOF`
+# against the Radius API. Diagnosing those runs requires distinguishing a
+# control-plane outage (etcd slowness, a kube-apiserver liveness kill) from a
+# Radius bug, and the pod snapshot alone cannot do that. This script captures
+# the missing evidence:
+#
+#   * pod state as YAML, which preserves `lastState.terminated.reason` and
+#     `exitCode` for containers that already restarted (`kubectl describe` has
+#     been observed to omit the `Last State` block entirely)
+#   * cluster-wide events, which record liveness/readiness probe failures and
+#     the resulting kills
+#   * node conditions, which record disk/memory/PID pressure
+#   * the current and previous logs of every kube-system container, including
+#     etcd - the previous instance's log is where an apiserver crash reason and
+#     etcd's `apply request took too long` / `slow fdatasync` warnings live
+#   * verbose /livez and /readyz output, which names the specific failing
+#     apiserver health check
+#
+# Collection is best effort by design: it runs after a test job that may have
+# already started tearing the cluster down, so an individual kubectl failure is
+# recorded in the output and never aborts the rest of the collection.
+# ============================================================================
+
+set -euo pipefail
+
+readonly SCRIPT_NAME="$(basename "$0")"
+
+OUTPUT_DIR=""
+PREFIX="cluster"
+
+usage() {
+    echo "Usage: ${SCRIPT_NAME} --output-dir DIR [--prefix NAME]"
+    echo "Options:"
+    echo "  -o, --output-dir   Directory to write diagnostics into (required)"
+    echo "  -p, --prefix       Filename prefix for the output files"
+    echo "                     (default: cluster)"
+    echo "  -h, --help         Show this help"
+    exit 0
+}
+
+validate_requirements() {
+    if [[ -z "${OUTPUT_DIR}" ]]; then
+        echo "Error: --output-dir is required" >&2
+        exit 1
+    fi
+
+    if ! command -v kubectl >/dev/null; then
+        echo "Error: kubectl is required but not installed" >&2
+        exit 1
+    fi
+}
+
+# capture appends the command line and its combined output to a file. A failing
+# command is recorded rather than propagated so one unavailable resource does
+# not stop the remaining collection.
+capture() {
+    local output_file="$1"
+    shift
+
+    {
+        echo "=============================================================="
+        echo "\$ $*"
+        echo "=============================================================="
+        "$@" 2>&1 || echo "(command exited with code $?)"
+        echo
+    } >>"${output_file}"
+}
+
+collect_pod_state() {
+    # The plain-text snapshot keeps the historical pod-states file intact, and
+    # the YAML dump adds the machine-readable lastState/exitCode fields.
+    capture "${OUTPUT_DIR}/${PREFIX}-pod-states.log" kubectl get pods -A -o wide
+    capture "${OUTPUT_DIR}/${PREFIX}-pod-states.log" kubectl describe pods -A
+    capture "${OUTPUT_DIR}/${PREFIX}-pods.yaml" kubectl get pods -A -o yaml
+}
+
+collect_events() {
+    capture "${OUTPUT_DIR}/${PREFIX}-events.log" \
+        kubectl get events -A --sort-by=.lastTimestamp
+}
+
+collect_node_state() {
+    capture "${OUTPUT_DIR}/${PREFIX}-nodes.log" kubectl get nodes -o wide
+    capture "${OUTPUT_DIR}/${PREFIX}-nodes.log" kubectl describe nodes
+}
+
+collect_control_plane_health() {
+    local output_file="${OUTPUT_DIR}/${PREFIX}-control-plane-health.log"
+
+    capture "${output_file}" kubectl get --raw "/livez?verbose"
+    capture "${output_file}" kubectl get --raw "/readyz?verbose"
+    capture "${output_file}" kubectl get apiservices -o wide
+}
+
+# collect_kube_system_logs writes the current and previous logs of every
+# kube-system container. Timestamps are included so the logs can be correlated
+# with Radius pod logs and with the test output.
+collect_kube_system_logs() {
+    local logs_dir="${OUTPUT_DIR}/${PREFIX}-kube-system-logs"
+    mkdir -p "${logs_dir}"
+
+    local pods
+    if ! pods="$(kubectl get pods -n kube-system \
+        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')"; then
+        echo "Warning: unable to list kube-system pods" >&2
+        return 0
+    fi
+
+    local pod container containers previous_file
+    for pod in ${pods}; do
+        if ! containers="$(kubectl get pod "${pod}" -n kube-system \
+            -o jsonpath='{range .spec.initContainers[*]}{.name}{"\n"}{end}{range .spec.containers[*]}{.name}{"\n"}{end}')"; then
+            continue
+        fi
+
+        for container in ${containers}; do
+            kubectl logs "${pod}" -n kube-system -c "${container}" \
+                --timestamps >"${logs_dir}/${pod}.${container}.log" 2>&1 || true
+
+            # A previous instance only exists once a container has restarted,
+            # and it holds the reason it died. Drop the file when there is no
+            # previous instance so the artifact only contains real restarts.
+            previous_file="${logs_dir}/${pod}.${container}.previous.log"
+            if ! kubectl logs "${pod}" -n kube-system -c "${container}" \
+                --previous --timestamps >"${previous_file}" 2>&1; then
+                rm -f "${previous_file}"
+            fi
+        done
+    done
+}
+
+main() {
+    validate_requirements
+
+    mkdir -p "${OUTPUT_DIR}"
+
+    echo "Collecting cluster diagnostics into ${OUTPUT_DIR}"
+    collect_pod_state
+    collect_events
+    collect_node_state
+    collect_control_plane_health
+    collect_kube_system_logs
+    echo "Cluster diagnostics collection complete"
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -o | --output-dir)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        -p | --prefix)
+            PREFIX="$2"
+            shift 2
+            ;;
+        -h | --help)
+            usage
+            ;;
+        *)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+    esac
+done
+
+main "$@"

.github/workflows/functional-test-cloud.yaml

@@ -1049,23 +1049,20 @@ jobs:
           result_directory: dist/functional_test/
           comment_mode: failures
 
-      - name: Collect Pod details
+      - name: Collect cluster diagnostics
         if: always()
-        # Diagnostic-only snapshot of cluster-wide pod state for post-mortem. A
-        # `kubectl describe pods -A` can race pod teardown and return a transient
-        # NotFound, so never let it fail the job.
+        # Diagnostic-only snapshot for post-mortem: pod state (including the
+        # previous container instance's termination reason), events, node
+        # conditions, control-plane health, and kube-system container logs.
+        # These commands can race cluster teardown and return a transient
+        # NotFound, so never let them fail the job.
         continue-on-error: true
         env:
           MATRIX_NAME: ${{ matrix.name }}
         run: |
-          POD_STATE_LOG_FILENAME="${RADIUS_CONTAINER_LOG_BASE}/${MATRIX_NAME}-tests-pod-states.log"
-          mkdir -p "$(dirname "${POD_STATE_LOG_FILENAME}")"
-          {
-            echo "kubectl get pods -A"
-            kubectl get pods -A
-            echo "kubectl describe pods -A"
-            kubectl describe pods -A
-          } >> "${POD_STATE_LOG_FILENAME}"
+          ./.github/scripts/collect-cluster-diagnostics.sh \
+            --output-dir "${RADIUS_CONTAINER_LOG_BASE}" \
+            --prefix "${MATRIX_NAME}-tests"
 
       - name: Upload container logs
         if: always()

.github/workflows/functional-test-noncloud.yaml

@@ -710,23 +710,20 @@ jobs:
           retention-days: 30
           if-no-files-found: error
 
-      - name: Collect Pod details
+      - name: Collect cluster diagnostics
         if: always()
-        # Diagnostic-only snapshot of cluster-wide pod state for post-mortem. A
-        # `kubectl describe pods -A` can race pod teardown and return a transient
-        # NotFound, so never let it fail the job.
+        # Diagnostic-only snapshot for post-mortem: pod state (including the
+        # previous container instance's termination reason), events, node
+        # conditions, control-plane health, and kube-system container logs.
+        # These commands can race cluster teardown and return a transient
+        # NotFound, so never let them fail the job.
         continue-on-error: true
         env:
           MATRIX_NAME: ${{ matrix.name }}
         run: |
-          POD_STATE_LOG_FILENAME="${RADIUS_CONTAINER_LOG_BASE}/${MATRIX_NAME}-tests-pod-states.log"
-          mkdir -p "$(dirname "$POD_STATE_LOG_FILENAME")"
-          {
-            echo "kubectl get pods -A"
-            kubectl get pods -A
-            echo "kubectl describe pods -A"
-            kubectl describe pods -A
-          }  >> "${POD_STATE_LOG_FILENAME}"
+          ./.github/scripts/collect-cluster-diagnostics.sh \
+            --output-dir "${RADIUS_CONTAINER_LOG_BASE}" \
+            --prefix "${MATRIX_NAME}-tests"
 
       - name: Upload container logs
         if: always()

.github/workflows/long-running-azure.yaml

@@ -451,21 +451,18 @@ jobs:
           make test-functional-all-cloud
           make test-functional-all-noncloud
 
-      - name: Collect Pod details
+      - name: Collect cluster diagnostics
         if: always()
-        # Diagnostic-only snapshot of cluster-wide pod state for post-mortem. A
-        # `kubectl describe pods -A` can race pod teardown and return a transient
-        # NotFound, so never let it fail the job.
+        # Diagnostic-only snapshot for post-mortem: pod state (including the
+        # previous container instance's termination reason), events, node
+        # conditions, control-plane health, and kube-system container logs.
+        # These commands can race cluster teardown and return a transient
+        # NotFound, so never let them fail the job.
         continue-on-error: true
         run: |
-          POD_STATE_LOG_FILENAME="${RADIUS_CONTAINER_LOG_BASE}/all-tests-pod-states.log"
-          mkdir -p $(dirname $POD_STATE_LOG_FILENAME)
-          {
-            echo "kubectl get pods -A"
-            kubectl get pods -A
-            echo "kubectl describe pods -A"
-            kubectl describe pods -A
-          } >> "${POD_STATE_LOG_FILENAME}"
+          ./.github/scripts/collect-cluster-diagnostics.sh \
+            --output-dir "${RADIUS_CONTAINER_LOG_BASE}" \
+            --prefix "all-tests"
 
       - name: Upload container logs
         if: always()

test/step/controlplane.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2023 The Radius Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package step
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	k8s "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+const (
+	// controlPlaneReadyPollInterval is how long to wait between readiness
+	// probes while the control plane is recovering.
+	controlPlaneReadyPollInterval = 5 * time.Second
+
+	// controlPlaneProbeTimeout bounds a single readiness probe so a hung
+	// kube-apiserver cannot consume the caller's entire retry budget in one
+	// request.
+	controlPlaneProbeTimeout = 10 * time.Second
+
+	// radiusAggregatedAPIPath is the aggregated API discovery path that rad
+	// itself requests when it opens a workspace connection (see
+	// pkg/cli/workspaces.Connection). Probing the same path exercises the whole
+	// chain the deployment depends on: the kube-apiserver, its aggregation
+	// layer, and the UCP APIService behind it.
+	radiusAggregatedAPIPath = "/apis/api.ucp.dev/v1alpha3"
+)
+
+// newControlPlaneReadyWaiter returns a function that blocks until the
+// Kubernetes control plane and the Radius aggregated API are both serving
+// again, or until ctx is done.
+//
+// It exists because a fixed sleep is not a useful gate between deployment
+// retries. When the kind control plane goes down, every queued retry fails
+// instantly against a dead socket and burns the retry budget in seconds while
+// the outage lasts minutes. Waiting for readiness instead makes each retry
+// attempt meaningful.
+//
+// It returns nil when no client is available, which disables the gate and
+// leaves the retry loop with its delay-only behavior.
+func newControlPlaneReadyWaiter(t *testing.T, client k8s.Interface) func(context.Context) error {
+	if client == nil {
+		return nil
+	}
+
+	restClient := client.Discovery().RESTClient()
+	if restClient == nil {
+		return nil
+	}
+
+	return func(ctx context.Context) error {
+		return waitForControlPlaneReady(ctx, t, restClient)
+	}
+}
+
+// waitForControlPlaneReady polls until the control plane is ready or ctx is
+// done. The error returned on timeout includes the last probe failure so the
+// test log records which component was still unavailable.
+func waitForControlPlaneReady(ctx context.Context, t *testing.T, client rest.Interface) error {
+	var lastErr error
+	for {
+		lastErr = checkControlPlaneReady(ctx, client)
+		if lastErr == nil {
+			return nil
+		}
+
+		t.Logf("waiting for the Radius control plane to become ready: %v", lastErr)
+
+		timer := time.NewTimer(controlPlaneReadyPollInterval)
+		select {
+		case <-timer.C:
+		case <-ctx.Done():
+			timer.Stop()
+			return fmt.Errorf("control plane did not become ready: %w (last probe failure: %v)", ctx.Err(), lastErr)
+		}
+	}
+}
+
+// checkControlPlaneReady performs a single readiness probe.
+func checkControlPlaneReady(ctx context.Context, client rest.Interface) error {
+	// /readyz reports whether the kube-apiserver has finished starting and its
+	// health checks - including the etcd backend check - are passing. It is
+	// served to any authenticated caller by the built-in
+	// system:public-info-viewer role, so it needs no extra RBAC.
+	if err := probe(ctx, client, "/readyz"); err != nil {
+		return fmt.Errorf("kube-apiserver is not ready: %w", err)
+	}
+
+	// The kube-apiserver can be ready before the aggregated API it proxies to
+	// is, and a request to an unavailable APIService fails with 503. Probing it
+	// avoids retrying a deployment that is guaranteed to fail.
+	if err := probe(ctx, client, radiusAggregatedAPIPath); err != nil {
+		return fmt.Errorf("the Radius aggregated API is not reachable: %w", err)
+	}
+
+	return nil
+}
+
+func probe(ctx context.Context, client rest.Interface, path string) error {
+	probeCtx, cancel := context.WithTimeout(ctx, controlPlaneProbeTimeout)
+	defer cancel()
+
+	return client.Get().AbsPath(path).Do(probeCtx).Error()
+}