Change cloudE2E test to use oblt-cli

.buildkite/pipeline.yml

@@ -34,6 +34,16 @@ common:
         lifetime: 10800 # seconds
         project-id: "elastic-observability-ci"
         project-number: "911195782929"
+  # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+  # This plugin authenticates to Google Cloud for oblt-cli access.
+  - google_oidc_observability_plugin: &google_oidc_observability_plugin
+      elastic/oblt-google-auth#v1.3.0:
+        lifetime: 10800 # seconds
+  - oblt_cli_plugin: &oblt_cli_plugin
+      elastic/oblt-cli#v0.4.0:
+        version-file: .oblt-cli-version
+  - vault_github_token: &vault_github_token
+      elastic/vault-github-token#v0.1.0:
 
 steps:
   - group: "Check and build"
@@ -264,16 +274,14 @@ steps:
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
           SNAPSHOT: "true"
           PLATFORMS: "linux/amd64"
-          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
         plugins:
           - *docker_elastic_login_plugin
-          - elastic/vault-secrets#v0.1.0:
-              path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
-              field: "apiKey"
-              env_var: "EC_API_KEY"
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
+          - *vault_github_token
         depends_on:
           - step: "unit-test"
             allow_failure: false
@@ -295,24 +303,15 @@ steps:
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
           SNAPSHOT: "true"
           PLATFORMS: "linux/amd64"
-          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
           FIPS: "true"
-          # Cloud e2e test is temporarily redirected to prod as new deployments are not able to start on FRH-staging.
-          # The following env vars should be uncommented once FRH-staging is fixed.
-          #EC_ENDPOINT: "https://api.staging.elastic-gov.com"
-          #TF_VAR_ess_region: "us-gov-east-1"
-          #TF_VAR_deployment_template_id: "aws-general-purpose"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
         plugins:
           - *docker_elastic_login_plugin
-          - elastic/vault-secrets#v0.1.0:
-              path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
-              # Use the commented path below instead of the one above when FRH-staging is fixed.
-              #path: "kv/ci-shared/platform-ingest/platform-ingest-ec-staging-gov"
-              field: "apiKey"
-              env_var: "EC_API_KEY"
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
+          - *vault_github_token
         depends_on:
           - step: "unit-test"
             allow_failure: false

.buildkite/scripts/cloud_e2e_test.sh

@@ -8,41 +8,78 @@ add_bin_path
 
 with_go
 
-with_Terraform
-
 with_docker_compose
 
-with_mage
+# Extract stack version from dev-tools/integration/.env.
+# Mirrors the regex logic in the former terraform main.tf:
+# ELASTICSEARCH_VERSION=9.5.0-<hash>-SNAPSHOT -> 9.5.0-SNAPSHOT
+ES_VERSION=$(grep "^ELASTICSEARCH_VERSION=" dev-tools/integration/.env | cut -d= -f2)
+if [[ "$ES_VERSION" == *SNAPSHOT* ]]; then
+  STACK_VERSION=$(echo "$ES_VERSION" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)-SNAPSHOT
+else
+  STACK_VERSION=$(echo "$ES_VERSION" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
+fi
 
 cleanup() {
   r=$?
-
-  if [ -f dev-tools/cloud/terraform/.terraform.lock.hcl ] ; then
-    echo "--- Deployment detected, running cleanup."
-    mage test:cloudE2EDown
+  if [ -f cluster-info.json ]; then
+    CLUSTER_NAME=$(jq -r '.ClusterName' cluster-info.json)
+    if [ -n "${CLUSTER_NAME}" ] && [ "${CLUSTER_NAME}" != "null" ]; then
+      echo "--- Deployment detected, running cleanup."
+      oblt-cli cluster destroy --cluster-name "${CLUSTER_NAME}" --force || true
+    fi
   else
-      echo "--- No deployment detected, skipping cleanup."
+    echo "--- No deployment detected, skipping cleanup."
   fi
   exit $r
 }
 trap cleanup EXIT INT TERM
 
-USER=fleetserverci mage docker:cover docker:customAgentImage docker:push test:cloudE2EUp
-FLEET_SERVER_URL=$(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate fleet_url)
-echo "Fleet server: \"${FLEET_SERVER_URL}\""
-echo "Deployment ID: $(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate deployment_id)"
+USER=fleetserverci mage docker:cover docker:customAgentImage docker:push
+
+echo "--- Provisioning cloud deployment (stack: ${STACK_VERSION})"
+oblt-cli cluster create custom \
+  --template ess-ea-it \
+  --cluster-name-prefix fleet-server \
+  --output-file="${PWD}/cluster-info.json" \
+  --wait 20 \
+  --parameter "StackVersion=${STACK_VERSION}" \
+  --parameter "ExpireInHours=2" \
+  --parameter "ElasticAgentDockerImage=${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}"
+
+CLUSTER_NAME=$(jq -r '.ClusterName' cluster-info.json)
+echo "Cluster: ${CLUSTER_NAME}"
+
+# Load deployment credentials as env vars.
+# oblt-cli exports vars with the prefixes: ELASTICSEARCH_*, KIBANA_*, FLEET_SERVER_*, INTEGRATIONS_SERVER_*
+oblt-cli cluster secrets env \
+  --cluster-name="${CLUSTER_NAME}" \
+  --output-file=secrets.env.sh
+set -a
+# shellcheck source=/dev/null
+source secrets.env.sh
+set +a
+rm -f secrets.env.sh
+
+# Map oblt-cli output var names to the names expected by testing/cloude2e/cloude2e_test.go
+export FLEET_SERVER_URL="${FLEET_SERVER_HOST:-}"
+export KIBANA_URL="${KIBANA_HOST:-}"
+export ELASTIC_USER="${ELASTICSEARCH_USERNAME:-}"
+export ELASTIC_PASS="${ELASTICSEARCH_PASSWORD:-}"
 
 if [[ "${FLEET_SERVER_URL}" == "" ]]; then
-    message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed"
-    if [[ "${CI}" == "true" ]]; then
-        buildkite-agent annotate \
-            "${message}" \
-            --context "ctx-cloude2e-test" \
-            --style "error"
-    fi
-    echo "${message}"
-    exit 1
+  message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed"
+  if [[ "${CI:-}" == "true" ]]; then
+    buildkite-agent annotate \
+      "${message}" \
+      --context "ctx-cloude2e-test" \
+      --style "error"
+  fi
+  echo "${message}"
+  exit 1
 fi
 
+echo "Fleet server: ${FLEET_SERVER_URL}"
+
 echo "--- Trigger cloud E2E test"
 mage test:cloudE2ERun

.oblt-cli-version

@@ -0,0 +1 @@
+8.0.12

magefile.go

@@ -2279,24 +2279,31 @@ func (Test) CloudE2EDown() error {
 }
 
 // CloudE2ERun runs tests against the remote cloud deployment.
+// It reads credentials from env vars (FLEET_SERVER_URL, KIBANA_URL, ELASTIC_USER, ELASTIC_PASS).
+// In CI, cloud_e2e_test.sh populates these from oblt-cli cluster secrets before calling this target.
+// Locally, if the env vars are absent, it falls back to reading from the terraform state written by CloudE2EUp.
 func (Test) CloudE2ERun() error {
-	fleetURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "fleet_url")
-	if err != nil {
-		return fmt.Errorf("unable to retrive fleet-server cloud url: %w", err)
-	}
-
-	kibanaURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "kibana_url")
-	if err != nil {
-		return fmt.Errorf("unable to retrive kibana cloud url: %w", err)
-	}
-
-	user, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_username")
-	if err != nil {
-		return fmt.Errorf("unable to retrive es username: %w", err)
-	}
-	pass, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_password")
-	if err != nil {
-		return fmt.Errorf("unable to retrive es password: %w", err)
+	fleetURL := os.Getenv("FLEET_SERVER_URL")
+	kibanaURL := os.Getenv("KIBANA_URL")
+	user := os.Getenv("ELASTIC_USER")
+	pass := os.Getenv("ELASTIC_PASS")
+
+	if fleetURL == "" || kibanaURL == "" || user == "" || pass == "" {
+		log.Println("Credential env vars not set, falling back to terraform state")
+		tfState := filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate")
+		var err error
+		if fleetURL, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "fleet_url"); err != nil {
+			return fmt.Errorf("unable to retrieve fleet-server cloud url: %w", err)
+		}
+		if kibanaURL, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "kibana_url"); err != nil {
+			return fmt.Errorf("unable to retrieve kibana cloud url: %w", err)
+		}
+		if user, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "elasticsearch_username"); err != nil {
+			return fmt.Errorf("unable to retrieve es username: %w", err)
+		}
+		if pass, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "elasticsearch_password"); err != nil {
+			return fmt.Errorf("unable to retrieve es password: %w", err)
+		}
 	}
 
 	var b bytes.Buffer
@@ -2311,7 +2318,7 @@ func (Test) CloudE2ERun() error {
 	)
 	cmd.Stdout = w
 	cmd.Stderr = w
-	err = cmd.Run()
+	err := cmd.Run()
 	err = errors.Join(err, os.WriteFile(filepath.Join("build", "test-cloude2e.out"), b.Bytes(), 0o644))
 	return err
 }