diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ec3f10e012..e9cda34bb8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -34,6 +34,16 @@ common: lifetime: 10800 # seconds project-id: "elastic-observability-ci" project-number: "911195782929" + # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf + # This plugin authenticates to Google Cloud for oblt-cli access. + - google_oidc_observability_plugin: &google_oidc_observability_plugin + elastic/oblt-google-auth#v1.3.1: + lifetime: 10800 # seconds + - oblt_cli_plugin: &oblt_cli_plugin + elastic/oblt-cli#v0.4.2: + version-file: .oblt-cli-version + - vault_github_token: &vault_github_token + elastic/vault-github-token#v0.2.1: steps: - group: "Check and build" @@ -264,16 +274,14 @@ steps: DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}" SNAPSHOT: "true" PLATFORMS: "linux/amd64" - TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}" command: ".buildkite/scripts/cloud_e2e_test.sh" agents: provider: "gcp" plugins: - *docker_elastic_login_plugin - - elastic/vault-secrets#v0.1.0: - path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod" - field: "apiKey" - env_var: "EC_API_KEY" + - *google_oidc_observability_plugin + - *oblt_cli_plugin + - *vault_github_token depends_on: - step: "unit-test" allow_failure: false @@ -295,24 +303,15 @@ steps: DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}" SNAPSHOT: "true" PLATFORMS: "linux/amd64" - TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}" FIPS: "true" - # Cloud e2e test is temporarily redirected to prod as new deployments are not able to start on FRH-staging. - # The following env vars should be uncommented once FRH-staging is fixed. - #EC_ENDPOINT: "https://api.staging.elastic-gov.com" - #TF_VAR_ess_region: "us-gov-east-1" - #TF_VAR_deployment_template_id: "aws-general-purpose" command: ".buildkite/scripts/cloud_e2e_test.sh" agents: provider: "gcp" plugins: - *docker_elastic_login_plugin - - elastic/vault-secrets#v0.1.0: - path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod" - # Use the commented path below instead of the one above when FRH-staging is fixed. - #path: "kv/ci-shared/platform-ingest/platform-ingest-ec-staging-gov" - field: "apiKey" - env_var: "EC_API_KEY" + - *google_oidc_observability_plugin + - *oblt_cli_plugin + - *vault_github_token depends_on: - step: "unit-test" allow_failure: false diff --git a/.buildkite/scripts/cloud_e2e_test.sh b/.buildkite/scripts/cloud_e2e_test.sh index 2482c7d48f..61af922a9d 100755 --- a/.buildkite/scripts/cloud_e2e_test.sh +++ b/.buildkite/scripts/cloud_e2e_test.sh @@ -8,41 +8,78 @@ add_bin_path with_go -with_Terraform - with_docker_compose with_mage +# STACK_BUILD_ID is the full pinned version from .env, e.g. 9.5.0-335b21fa-SNAPSHOT. +# It is used to pin ES and Kibana images to the exact build being tested. +STACK_BUILD_ID=$(grep "^ELASTICSEARCH_VERSION=" dev-tools/integration/.env | cut -d= -f2) +# STACK_VERSION strips the build hash for the oblt-cli StackVersion parameter, e.g. 9.5.0-SNAPSHOT. +if [[ "$STACK_BUILD_ID" == *SNAPSHOT* ]]; then + STACK_VERSION=$(echo "$STACK_BUILD_ID" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)-SNAPSHOT +else + STACK_VERSION=$(echo "$STACK_BUILD_ID" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) +fi + cleanup() { r=$? - - if [ -f dev-tools/cloud/terraform/.terraform.lock.hcl ] ; then - echo "--- Deployment detected, running cleanup." - mage test:cloudE2EDown + if [ -f cluster-info.json ]; then + CLUSTER_NAME=$(jq -r '.ClusterName' cluster-info.json) + if [ -n "${CLUSTER_NAME}" ] && [ "${CLUSTER_NAME}" != "null" ]; then + echo "--- Deployment detected, running cleanup." + oblt-cli cluster destroy --cluster-name "${CLUSTER_NAME}" --force || true + fi else - echo "--- No deployment detected, skipping cleanup." + echo "--- No deployment detected, skipping cleanup." fi exit $r } trap cleanup EXIT INT TERM -USER=fleetserverci mage docker:cover docker:customAgentImage docker:push test:cloudE2EUp -FLEET_SERVER_URL=$(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate fleet_url) -echo "Fleet server: \"${FLEET_SERVER_URL}\"" -echo "Deployment ID: $(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate deployment_id)" - -if [[ "${FLEET_SERVER_URL}" == "" ]]; then - message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed" - if [[ "${CI}" == "true" ]]; then - buildkite-agent annotate \ - "${message}" \ - --context "ctx-cloude2e-test" \ - --style "error" - fi - echo "${message}" - exit 1 +USER=fleetserverci mage docker:cover docker:customAgentImage docker:push + +echo "--- Provisioning cloud deployment (stack: ${STACK_VERSION})" +oblt-cli cluster create custom \ + --template ess-ea-it \ + --cluster-name-prefix fleet-server \ + --output-file="${PWD}/cluster-info.json" \ + --wait 20 \ + --parameter "StackVersion=${STACK_VERSION}" \ + --parameter "ExpireInHours=2" \ + --parameter "ElasticAgentDockerImage=${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" \ + --parameter "ElasticsearchDockerImage=docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${STACK_BUILD_ID}" \ + --parameter "KibanaDockerImage=docker.elastic.co/cloud-release/kibana-cloud:${STACK_BUILD_ID}" \ + --parameter "ElasticTeam=elastic-agent-control-plane" \ + --parameter "ElasticProject=fleet-server-ci" + +CLUSTER_NAME=$(jq -r '.ClusterName' cluster-info.json) +echo "Cluster: ${CLUSTER_NAME}" + +# Load deployment credentials as env vars. +# oblt-cli exports: FLEET_URL, KIBANA_HOST, ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD, and others. +oblt-cli cluster secrets env \ + --cluster-name="${CLUSTER_NAME}" \ + --output-file=secrets.env.sh +set -a +# shellcheck source=/dev/null +source secrets.env.sh +set +a +rm -f secrets.env.sh + +if [[ "${FLEET_URL:-}" == "" ]]; then + message="FLEET_URL is empty, cloud e2e tests cannot be executed" + if [[ "${CI:-}" == "true" ]]; then + buildkite-agent annotate \ + "${message}" \ + --context "ctx-cloude2e-test" \ + --style "error" + fi + echo "${message}" + exit 1 fi +echo "Fleet server: ${FLEET_URL}" + echo "--- Trigger cloud E2E test" mage test:cloudE2ERun diff --git a/.oblt-cli-version b/.oblt-cli-version new file mode 100644 index 0000000000..483e728539 --- /dev/null +++ b/.oblt-cli-version @@ -0,0 +1 @@ +8.0.13 diff --git a/magefile.go b/magefile.go index 1c2b0c1a3f..71170ccecc 100644 --- a/magefile.go +++ b/magefile.go @@ -2279,24 +2279,32 @@ func (Test) CloudE2EDown() error { } // CloudE2ERun runs tests against the remote cloud deployment. +// It reads credentials from env vars matching those set by `oblt-cli cluster secrets env`: +// FLEET_URL, KIBANA_HOST, ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD. +// In CI, cloud_e2e_test.sh populates these before calling this target. +// Locally, if the env vars are absent, it falls back to reading from the terraform state written by CloudE2EUp. func (Test) CloudE2ERun() error { - fleetURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "fleet_url") - if err != nil { - return fmt.Errorf("unable to retrive fleet-server cloud url: %w", err) - } - - kibanaURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "kibana_url") - if err != nil { - return fmt.Errorf("unable to retrive kibana cloud url: %w", err) - } - - user, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_username") - if err != nil { - return fmt.Errorf("unable to retrive es username: %w", err) - } - pass, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_password") - if err != nil { - return fmt.Errorf("unable to retrive es password: %w", err) + fleetURL := os.Getenv("FLEET_URL") + kibanaURL := os.Getenv("KIBANA_HOST") + user := os.Getenv("ELASTICSEARCH_USERNAME") + pass := os.Getenv("ELASTICSEARCH_PASSWORD") + + if fleetURL == "" || kibanaURL == "" || user == "" || pass == "" { + log.Println("Credential env vars not set, falling back to terraform state") + tfState := filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate") + var err error + if fleetURL, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "fleet_url"); err != nil { + return fmt.Errorf("unable to retrieve fleet-server cloud url: %w", err) + } + if kibanaURL, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "kibana_url"); err != nil { + return fmt.Errorf("unable to retrieve kibana cloud url: %w", err) + } + if user, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "elasticsearch_username"); err != nil { + return fmt.Errorf("unable to retrieve es username: %w", err) + } + if pass, err = sh.Output("terraform", "output", "--raw", "--state="+tfState, "elasticsearch_password"); err != nil { + return fmt.Errorf("unable to retrieve es password: %w", err) + } } var b bytes.Buffer @@ -2304,14 +2312,14 @@ func (Test) CloudE2ERun() error { cmd := exec.Command("go", "test", "-v", "-timeout", "30m", "-tags=cloude2e", "-count=1", "-p", "1", "./...") cmd.Dir = "testing" cmd.Env = append(os.Environ(), - "FLEET_SERVER_URL="+fleetURL, - "KIBANA_URL="+kibanaURL, - "ELASTIC_USER="+user, - "ELASTIC_PASS="+pass, + "FLEET_URL="+fleetURL, + "KIBANA_HOST="+kibanaURL, + "ELASTICSEARCH_USERNAME="+user, + "ELASTICSEARCH_PASSWORD="+pass, ) cmd.Stdout = w cmd.Stderr = w - err = cmd.Run() + err := cmd.Run() err = errors.Join(err, os.WriteFile(filepath.Join("build", "test-cloude2e.out"), b.Bytes(), 0o644)) return err } diff --git a/testing/cloude2e/cloude2e_test.go b/testing/cloude2e/cloude2e_test.go index b510d4a70a..c2b6516687 100644 --- a/testing/cloude2e/cloude2e_test.go +++ b/testing/cloude2e/cloude2e_test.go @@ -40,21 +40,21 @@ type TestSuite struct { func (suite *TestSuite) SetupSuite() { suite.T().Helper() - // get env vars - v, ok := os.LookupEnv("FLEET_SERVER_URL") - suite.Require().True(ok, "expected FLEET_SERVER_URL to be defined") + // get env vars — names match those set by `oblt-cli cluster secrets env` + v, ok := os.LookupEnv("FLEET_URL") + suite.Require().True(ok, "expected FLEET_URL to be defined") suite.fleetServerURL = v - v, ok = os.LookupEnv("KIBANA_URL") - suite.Require().True(ok, "expected KIBANA_URL to be defined") + v, ok = os.LookupEnv("KIBANA_HOST") + suite.Require().True(ok, "expected KIBANA_HOST to be defined") suite.kibanaURL = v - v, ok = os.LookupEnv("ELASTIC_USER") - suite.Require().True(ok, "expected ELASTIC_USER to be defined") + v, ok = os.LookupEnv("ELASTICSEARCH_USERNAME") + suite.Require().True(ok, "expected ELASTICSEARCH_USERNAME to be defined") suite.username = v - v, ok = os.LookupEnv("ELASTIC_PASS") - suite.Require().True(ok, "expected ELASTIC_PASS to be defined") + v, ok = os.LookupEnv("ELASTICSEARCH_PASSWORD") + suite.Require().True(ok, "expected ELASTICSEARCH_PASSWORD to be defined") suite.password = v suite.client = &http.Client{}