Move private-cluster integration test to 1ES pipeline

.azure-pipelines/1es-integration-tests-private.yml

@@ -0,0 +1,224 @@
+# Private-cluster release gate (1ES pipeline).
+#
+# Runs the k8s-deploy action against a real private AKS cluster before a release
+# is published. The internal release orchestrator must queue this pipeline for
+# the exact releases/vX.Y.Z commit, pass that SHA as releaseCandidateCommit, and
+# depend on this run succeeding before it tags or publishes the release.
+#
+# This pipeline intentionally has no CI or PR trigger. It uses a governed WIF
+# service connection and must never execute code supplied by a pull request.
+#
+# The node24 JS action is invoked as `node lib/index.js` with inputs supplied via
+# INPUT_* env vars. There is no action.yml default injection here, so keep INPUT_*
+# in sync with action.yml; skip-tls-verify (getBooleanInput) and GITHUB_WORKFLOW
+# must be set or the action throws.
+#
+# The pipeline does not create/delete the resource group (it must already exist);
+# it only creates and deletes a per-build cluster inside it. Azure DevOps pipeline
+# UI triggers must also remain disabled because UI settings override this YAML.
+# The service connection must use a custom role scoped to the shared resource
+# group; do not grant subscription-wide Contributor or Owner.
+
+parameters:
+   - name: releaseCandidateCommit
+     displayName: Exact release candidate commit SHA
+     type: string
+     default: ''
+
+trigger: none
+pr: none
+
+resources:
+   repositories:
+      - repository: 1esPipelines
+        type: git
+        name: 1ESPipelineTemplates/1ESPipelineTemplates
+        ref: refs/tags/release
+
+variables:
+   serviceConnection: k8s-deploy-intg-test-svc-conn
+   resourceGroup: k8s-deploy-intg-rg
+   location: eastus2
+   kubectlVersion: 1.34.1
+   clusterName: test-$(Build.BuildId)
+   namespace: test-$(Build.BuildId)
+   workflowFriendlyName: k8s-deploy-integration-tests-private
+
+extends:
+   template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines
+   parameters:
+      pool:
+         name: staging-pool-amd64-mariner-2
+         image: 1es-azlinux-3-amd64-custom-disk
+         os: linux
+         hostArchitecture: amd64
+      sdl:
+         sourceAnalysisPool:
+            name: staging-pool-amd64-mariner-2
+            image: azcu-agent-amd64-windows-22-img
+            os: windows
+            hostArchitecture: amd64
+      stages:
+         - stage: integration_test
+           displayName: Private cluster integration tests
+           jobs:
+              - job: run_integration_test
+                displayName: Run Private Cluster Integration Tests
+                timeoutInMinutes: 60
+                steps:
+                   - task: NodeTool@0
+                     displayName: Install Node.js
+                     inputs:
+                        versionSpec: '24.x'
+
+                   - script: |
+                        set -euo pipefail
+
+                        if [[ ! "$BUILD_SOURCEBRANCH" =~ ^refs/heads/releases/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+                           echo "##vso[task.logissue type=error]This release gate only accepts releases/vX.Y.Z branches; received $BUILD_SOURCEBRANCH"
+                           exit 1
+                        fi
+
+                        if [[ ! "$EXPECTED_RELEASE_COMMIT" =~ ^[0-9a-fA-F]{40}$ ]]; then
+                           echo "##vso[task.logissue type=error]releaseCandidateCommit must be a full 40-character commit SHA"
+                           exit 1
+                        fi
+
+                        checked_out_commit="$(git rev-parse HEAD)"
+                        if [[ "$BUILD_SOURCEVERSION" != "$EXPECTED_RELEASE_COMMIT" || "$checked_out_commit" != "$EXPECTED_RELEASE_COMMIT" ]]; then
+                           echo "##vso[task.logissue type=error]Release candidate mismatch: requested $EXPECTED_RELEASE_COMMIT, source $BUILD_SOURCEVERSION, checked out $checked_out_commit"
+                           exit 1
+                        fi
+
+                        expected_version="${BUILD_SOURCEBRANCH#refs/heads/releases/v}"
+                        actual_version="$(node -p "require('./package.json').version")"
+                        if [[ "$actual_version" != "$expected_version" ]]; then
+                           echo "##vso[task.logissue type=error]Release branch version $expected_version does not match package.json version $actual_version"
+                           exit 1
+                        fi
+
+                        echo "##vso[task.setvariable variable=releaseCandidateValidated]true"
+                     displayName: Validate release candidate
+                     env:
+                        EXPECTED_RELEASE_COMMIT: ${{ parameters.releaseCandidateCommit }}
+
+                   - script: npm ci
+                     displayName: Install dependencies
+
+                   - script: npm run build
+                     displayName: Build
+
+                   - task: KubectlInstaller@0
+                     displayName: Install kubectl
+                     inputs:
+                        kubectlVersion: $(kubectlVersion)
+
+                   - task: AzureCLI@2
+                     displayName: Create private AKS cluster and set context
+                     inputs:
+                        azureSubscription: $(serviceConnection)
+                        scriptType: bash
+                        scriptLocation: inlineScript
+                        inlineScript: |
+                           set -euo pipefail
+                           set +x
+                           # RG must already exist; do not run `az group create`.
+                           az aks create \
+                              --name $(clusterName) \
+                              --resource-group $(resourceGroup) \
+                              --location $(location) \
+                              --enable-private-cluster \
+                              --generate-ssh-keys \
+                              --node-count 1 \
+                              --tags purpose=k8s-deploy-private-integration build-id=$(Build.BuildId)
+                           az aks get-credentials --resource-group $(resourceGroup) --name $(clusterName)
+
+                   - task: AzureCLI@2
+                     displayName: Create namespace to run tests
+                     inputs:
+                        azureSubscription: $(serviceConnection)
+                        scriptType: bash
+                        scriptLocation: inlineScript
+                        inlineScript: |
+                           set -euo pipefail
+                           az aks command invoke --resource-group $(resourceGroup) --name $(clusterName) --command "kubectl create ns $(namespace)"
+
+                   - task: UsePythonVersion@0
+                     displayName: Install Python
+                     inputs:
+                        versionSpec: '3.x'
+
+                   - task: AzureCLI@2
+                     displayName: Executing deploy action for pod
+                     inputs:
+                        azureSubscription: $(serviceConnection)
+                        scriptType: bash
+                        scriptLocation: inlineScript
+                        inlineScript: |
+                           set -euo pipefail
+                           node lib/index.js
+                     env:
+                        KUBECONFIG: $(HOME)/.kube/config
+                        # Inputs supplied via INPUT_* (no action.yml injection here).
+                        INPUT_ACTION: deploy
+                        INPUT_STRATEGY: basic
+                        INPUT_TRAFFIC-SPLIT-METHOD: pod
+                        INPUT_SKIP-TLS-VERIFY: 'false'
+                        INPUT_NAMESPACE: $(namespace)
+                        INPUT_IMAGES: nginx:1.14.2
+                        INPUT_MANIFESTS: |
+                           test/integration/manifests/test.yml
+                           test/integration/manifests/test2.yml
+                        INPUT_PRIVATE-CLUSTER: 'true'
+                        INPUT_RESOURCE-GROUP: $(resourceGroup)
+                        INPUT_NAME: $(clusterName)
+                        INPUT_RESOURCE-TYPE: Microsoft.ContainerService/managedClusters
+                        INPUT_PULL-IMAGES: 'false'
+                        INPUT_ANNOTATE-RESOURCES: 'true'
+                        INPUT_ANNOTATE-NAMESPACE: 'true'
+                        INPUT_FORCE: 'false'
+                        INPUT_SERVER-SIDE: 'false'
+                        INPUT_TIMEOUT: 10m
+                        # Emulate the GitHub Action context used in annotations and
+                        # manifest path containment. A workflow-style path avoids an
+                        # unnecessary GitHub API request for its display name.
+                        GITHUB_WORKSPACE: $(Build.Repository.LocalPath)
+                        RUNNER_TEMP: $(Agent.TempDirectory)
+                        GITHUB_REPOSITORY: Azure/k8s-deploy
+                        GITHUB_SHA: $(Build.SourceVersion)
+                        GITHUB_REF: $(Build.SourceBranch)
+                        GITHUB_RUN_ID: $(Build.BuildId)
+                        GITHUB_JOB: run_integration_test
+                        GITHUB_ACTOR: $(Build.RequestedFor)
+                        GITHUB_WORKFLOW: .github/workflows/$(workflowFriendlyName)
+
+                   - task: AzureCLI@2
+                     displayName: Checking if deployments and services were created
+                     inputs:
+                        azureSubscription: $(serviceConnection)
+                        scriptType: bash
+                        scriptLocation: inlineScript
+                        inlineScript: |
+                           set -euo pipefail
+                           python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Deployment name=nginx-deployment containerName=nginx:1.14.2 labels=app:nginx,workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx
+                           python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Service name=nginx-service labels=workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx
+
+                           python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Deployment name=nginx-deployment2 containerName=nginx:1.14.2 labels=app:nginx2,workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx2
+                           python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Service name=nginx-service2 labels=workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx2
+
+                   - task: AzureCLI@2
+                     displayName: Clean up AKS cluster
+                     condition: and(always(), eq(variables['releaseCandidateValidated'], 'true'))
+                     inputs:
+                        azureSubscription: $(serviceConnection)
+                        scriptType: bash
+                        scriptLocation: inlineScript
+                        inlineScript: |
+                           set -euo pipefail
+                           # Delete only the cluster; the shared RG persists.
+                           if az aks show --resource-group $(resourceGroup) --name $(clusterName) >/dev/null 2>&1; then
+                              echo "deleting AKS cluster $(clusterName)"
+                              az aks delete --yes --resource-group $(resourceGroup) --name $(clusterName)
+                           else
+                              echo "AKS cluster $(clusterName) was not created; nothing to delete"
+                           fi

test/integration/k8s-deploy-test.py

@@ -21,6 +21,7 @@
 ingressServicesKey = "ingressServices"
 tsServicesKey = "tsServices"
 privateKey = "private"
+resourceGroupKey = "resourceGroup"
 
 
 def parseArgs(sysArgs):
@@ -211,7 +212,12 @@ def main():
     try:
         if privateKey in parsedArgs:
             uniqueName = parsedArgs[privateKey]
-            azPrefix = f"az aks command invoke --resource-group {uniqueName} --name {uniqueName} --command "
+            # The resource group defaults to the cluster's unique name for
+            # backward compatibility (older workflows named the RG and cluster
+            # identically). When the RG is shared/pre-created and differs from
+            # the cluster name, pass it explicitly via resourceGroup=<rg>.
+            resourceGroup = parsedArgs.get(resourceGroupKey, uniqueName)
+            azPrefix = f"az aks command invoke --resource-group {resourceGroup} --name {uniqueName} --command "
             cmd = azPrefix + "'" + cmd + "'"
             outputString = os.popen(cmd).read()
             successExit = "exitcode=0"