Change cloudE2E test to use oblt-cli

.buildkite/pipeline.yml

@@ -34,6 +34,16 @@ common:
         lifetime: 10800 # seconds
         project-id: "elastic-observability-ci"
         project-number: "911195782929"
+  # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+  # This plugin authenticates to Google Cloud for oblt-cli access.
+  - google_oidc_observability_plugin: &google_oidc_observability_plugin
+      elastic/oblt-google-auth#v1.3.1:
+        lifetime: 10800 # seconds
+  - oblt_cli_plugin: &oblt_cli_plugin
+      elastic/oblt-cli#v0.4.2:
+        version-file: .oblt-cli-version
+  - vault_github_token: &vault_github_token
+      elastic/vault-github-token#v0.2.1:
 
 steps:
   - group: "Check and build"
@@ -264,16 +274,14 @@ steps:
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
           SNAPSHOT: "true"
           PLATFORMS: "linux/amd64"
-          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
         plugins:
           - *docker_elastic_login_plugin
-          - elastic/vault-secrets#v0.1.0:
-              path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
-              field: "apiKey"
-              env_var: "EC_API_KEY"
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
+          - *vault_github_token
         depends_on:
           - step: "unit-test"
             allow_failure: false
@@ -295,24 +303,15 @@ steps:
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
           SNAPSHOT: "true"
           PLATFORMS: "linux/amd64"
-          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
           FIPS: "true"
-          # Cloud e2e test is temporarily redirected to prod as new deployments are not able to start on FRH-staging.
-          # The following env vars should be uncommented once FRH-staging is fixed.
-          #EC_ENDPOINT: "https://api.staging.elastic-gov.com"
-          #TF_VAR_ess_region: "us-gov-east-1"
-          #TF_VAR_deployment_template_id: "aws-general-purpose"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
         plugins:
           - *docker_elastic_login_plugin
-          - elastic/vault-secrets#v0.1.0:
-              path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
-              # Use the commented path below instead of the one above when FRH-staging is fixed.
-              #path: "kv/ci-shared/platform-ingest/platform-ingest-ec-staging-gov"
-              field: "apiKey"
-              env_var: "EC_API_KEY"
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
+          - *vault_github_token
         depends_on:
           - step: "unit-test"
             allow_failure: false

.buildkite/scripts/cloud_e2e_test.sh

@@ -8,41 +8,78 @@ add_bin_path
 
 with_go
 
-with_Terraform
-
 with_docker_compose
 
 with_mage
 
+# STACK_BUILD_ID is the full pinned version from .env, e.g. 9.5.0-335b21fa-SNAPSHOT.
+# It is used to pin ES and Kibana images to the exact build being tested.
+STACK_BUILD_ID=$(grep "^ELASTICSEARCH_VERSION=" dev-tools/integration/.env | cut -d= -f2)
+# STACK_VERSION strips the build hash for the oblt-cli StackVersion parameter, e.g. 9.5.0-SNAPSHOT.
+if [[ "$STACK_BUILD_ID" == *SNAPSHOT* ]]; then
+  STACK_VERSION=$(echo "$STACK_BUILD_ID" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)-SNAPSHOT
+else
+  STACK_VERSION=$(echo "$STACK_BUILD_ID" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
+fi
+
 cleanup() {
   r=$?
-
-  if [ -f dev-tools/cloud/terraform/.terraform.lock.hcl ] ; then
-    echo "--- Deployment detected, running cleanup."
-    mage test:cloudE2EDown
+  if [ -f cluster-info.json ]; then
+    CLUSTER_NAME=$(jq -r '.ClusterName' cluster-info.json)
+    if [ -n "${CLUSTER_NAME}" ] && [ "${CLUSTER_NAME}" != "null" ]; then
+      echo "--- Deployment detected, running cleanup."
+      oblt-cli cluster destroy --cluster-name "${CLUSTER_NAME}" --force || true
+    fi
   else
-      echo "--- No deployment detected, skipping cleanup."
+    echo "--- No deployment detected, skipping cleanup."
   fi
   exit $r
 }
 trap cleanup EXIT INT TERM
 
-USER=fleetserverci mage docker:cover docker:customAgentImage docker:push test:cloudE2EUp
-FLEET_SERVER_URL=$(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate fleet_url)
-echo "Fleet server: \"${FLEET_SERVER_URL}\""
-echo "Deployment ID: $(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate deployment_id)"
-
-if [[ "${FLEET_SERVER_URL}" == "" ]]; then
-    message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed"
-    if [[ "${CI}" == "true" ]]; then
-        buildkite-agent annotate \
-            "${message}" \
-            --context "ctx-cloude2e-test" \
-            --style "error"
-    fi
-    echo "${message}"
-    exit 1
+USER=fleetserverci mage docker:cover docker:customAgentImage docker:push
+
+echo "--- Provisioning cloud deployment (stack: ${STACK_VERSION})"
+oblt-cli cluster create custom \
+  --template ess-ea-it \
+  --cluster-name-prefix fleet-server \
+  --output-file="${PWD}/cluster-info.json" \
+  --wait 20 \
+  --parameter "StackVersion=${STACK_VERSION}" \
+  --parameter "ExpireInHours=2" \
+  --parameter "ElasticAgentDockerImage=${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" \
+  --parameter "ElasticsearchDockerImage=docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${STACK_BUILD_ID}" \
+  --parameter "KibanaDockerImage=docker.elastic.co/cloud-release/kibana-cloud:${STACK_BUILD_ID}" \
+  --parameter "ElasticTeam=elastic-agent-control-plane" \
+  --parameter "ElasticProject=fleet-server-ci"
+
+CLUSTER_NAME=$(jq -r '.ClusterName' cluster-info.json)
+echo "Cluster: ${CLUSTER_NAME}"
+
+# Load deployment credentials as env vars.
+# oblt-cli exports: FLEET_URL, KIBANA_HOST, ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD, and others.
+oblt-cli cluster secrets env \
+  --cluster-name="${CLUSTER_NAME}" \
+  --output-file=secrets.env.sh
+set -a
+# shellcheck source=/dev/null
+source secrets.env.sh
+set +a
+rm -f secrets.env.sh
+
+if [[ "${FLEET_URL:-}" == "" ]]; then
+  message="FLEET_URL is empty, cloud e2e tests cannot be executed"
+  if [[ "${CI:-}" == "true" ]]; then
+    buildkite-agent annotate \
+      "${message}" \
+      --context "ctx-cloude2e-test" \
+      --style "error"
+  fi
+  echo "${message}"
+  exit 1
 fi
 
+echo "Fleet server: ${FLEET_URL}"
+
 echo "--- Trigger cloud E2E test"
 mage test:cloudE2ERun

.oblt-cli-version

@@ -0,0 +1 @@
+8.0.13

magefile.go

@@ -2279,39 +2279,47 @@ func (Test) CloudE2EDown() error {
 }
 
 // CloudE2ERun runs tests against the remote cloud deployment.
+// It reads credentials from env vars matching those set by `oblt-cli cluster secrets env`:
+// FLEET_URL, KIBANA_HOST, ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD.
+// In CI, cloud_e2e_test.sh populates these before calling this target.
+// Locally, if the env vars are absent, it falls back to reading from the terraform state written by CloudE2EUp.
 func (Test) CloudE2ERun() error {
-	fleetURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "fleet_url")
-	if err != nil {
-		return fmt.Errorf("unable to retrive fleet-server cloud url: %w", err)
-	}
-
-	kibanaURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "kibana_url")
-	if err != nil {
-		return fmt.Errorf("unable to retrive kibana cloud url: %w", err)
-	}
-
-	user, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_username")
-	if err != nil {
-		return fmt.Errorf("unable to retrive es username: %w", err)
-	}
-	pass, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_password")
-	if err != nil {
-		return fmt.Errorf("unable to retrive es password: %w", err)
+	fleetURL := os.Getenv("FLEET_URL")
+	kibanaURL := os.Getenv("KIBANA_HOST")
+	user := os.Getenv("ELASTICSEARCH_USERNAME")
+	pass := os.Getenv("ELASTICSEARCH_PASSWORD")
+
+	if fleetURL == "" || kibanaURL == "" || user == "" || pass == "" {
+		log.Println("Credential env vars not set, falling back to terraform state")
+		tfState := filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate")
+		var err error
+		if fleetURL, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "fleet_url"); err != nil {
+			return fmt.Errorf("unable to retrieve fleet-server cloud url: %w", err)
+		}
+		if kibanaURL, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "kibana_url"); err != nil {
+			return fmt.Errorf("unable to retrieve kibana cloud url: %w", err)
+		}
+		if user, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "elasticsearch_username"); err != nil {
+			return fmt.Errorf("unable to retrieve es username: %w", err)
+		}
+		if pass, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "elasticsearch_password"); err != nil {
+			return fmt.Errorf("unable to retrieve es password: %w", err)
+		}
 	}
 
 	var b bytes.Buffer
 	w := io.MultiWriter(&b, os.Stdout)
 	cmd := exec.Command("go", "test", "-v", "-timeout", "30m", "-tags=cloude2e", "-count=1", "-p", "1", "./...")
 	cmd.Dir = "testing"
 	cmd.Env = append(os.Environ(),
-		"FLEET_SERVER_URL="+fleetURL,
-		"KIBANA_URL="+kibanaURL,
-		"ELASTIC_USER="+user,
-		"ELASTIC_PASS="+pass,
+		"FLEET_URL="+fleetURL,
+		"KIBANA_HOST="+kibanaURL,
+		"ELASTICSEARCH_USERNAME="+user,
+		"ELASTICSEARCH_PASSWORD="+pass,
 	)
 	cmd.Stdout = w
 	cmd.Stderr = w
-	err = cmd.Run()
+	err := cmd.Run()
 	err = errors.Join(err, os.WriteFile(filepath.Join("build", "test-cloude2e.out"), b.Bytes(), 0o644))
 	return err
 }

testing/cloude2e/cloude2e_test.go

@@ -40,21 +40,21 @@ type TestSuite struct {
 func (suite *TestSuite) SetupSuite() {
 	suite.T().Helper()
 
-	// get env vars
-	v, ok := os.LookupEnv("FLEET_SERVER_URL")
-	suite.Require().True(ok, "expected FLEET_SERVER_URL to be defined")
+	// get env vars — names match those set by `oblt-cli cluster secrets env`
+	v, ok := os.LookupEnv("FLEET_URL")
+	suite.Require().True(ok, "expected FLEET_URL to be defined")
 	suite.fleetServerURL = v
 
-	v, ok = os.LookupEnv("KIBANA_URL")
-	suite.Require().True(ok, "expected KIBANA_URL to be defined")
+	v, ok = os.LookupEnv("KIBANA_HOST")
+	suite.Require().True(ok, "expected KIBANA_HOST to be defined")
 	suite.kibanaURL = v
 
-	v, ok = os.LookupEnv("ELASTIC_USER")
-	suite.Require().True(ok, "expected ELASTIC_USER to be defined")
+	v, ok = os.LookupEnv("ELASTICSEARCH_USERNAME")
+	suite.Require().True(ok, "expected ELASTICSEARCH_USERNAME to be defined")
 	suite.username = v
 
-	v, ok = os.LookupEnv("ELASTIC_PASS")
-	suite.Require().True(ok, "expected ELASTIC_PASS to be defined")
+	v, ok = os.LookupEnv("ELASTICSEARCH_PASSWORD")
+	suite.Require().True(ok, "expected ELASTICSEARCH_PASSWORD to be defined")
 	suite.password = v
 
 	suite.client = &http.Client{}