diff --git a/ci/dockerfiles/main-postgres/Dockerfile b/ci/dockerfiles/main-postgres/Dockerfile index d58dc64d9c2..6be532e25a5 100644 --- a/ci/dockerfiles/main-postgres/Dockerfile +++ b/ci/dockerfiles/main-postgres/Dockerfile @@ -18,4 +18,7 @@ RUN echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg m "postgresql-${DB_VERSION}" \ "postgresql-client-${DB_VERSION}" -RUN sed -i 's/port = 5433/port = 5432/g' "/etc/postgresql/${DB_VERSION}/main/postgresql.conf" +RUN config="/etc/postgresql/${DB_VERSION}/main/postgresql.conf"; \ + if [ -f "$config" ]; then \ + sed -i 's/port = 5433/port = 5432/g' "$config"; \ + fi diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 8fe3f7c922a..16650b17925 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -12,7 +12,7 @@ groups: - unit-release - unit-director-sqlite - unit-director-mysql-8 - - unit-director-postgres-13 + - unit-director-postgres-17 - unit-director-postgres-15 - run-bundle-audit - integration-postgres @@ -25,6 +25,10 @@ groups: - bats-fips - bats-fips-cleanup-leftovers - brats-acceptance + - brats-postgres-15 + - brats-postgres-16 + - brats-postgres-17 + - brats-postgres-18 - brats-utils - brats-performance - bosh-disaster-recovery-acceptance-tests @@ -44,7 +48,7 @@ groups: - name: container-images jobs: - - build-main-postgres-13 + - build-main-postgres-17 - build-main-postgres-15 - build-main-mysql-8-0 - build-integration @@ -67,7 +71,7 @@ jobs: - unit-release - unit-director-sqlite - unit-director-mysql-8 - - unit-director-postgres-13 + - unit-director-postgres-17 - unit-director-postgres-15 - run-bundle-audit plan: @@ -202,10 +206,10 @@ jobs: DB: mysql RAKE_TASK: spec:unit:director - - name: unit-director-postgres-13 + - name: unit-director-postgres-17 public: true serial: true - serial_groups: [ unit-director-postgres-13 ] + serial_groups: [ unit-director-postgres-17 ] build_log_retention: builds: 250 plan: @@ -214,16 +218,17 @@ jobs: - get: bosh passed: [ gate ] trigger: true - - get: integration-postgres-13-image + - get: integration-postgres-17-image - task: test-rake-task file: bosh-ci/ci/tasks/test-rake-task.yml - image: integration-postgres-13-image + image: integration-postgres-17-image timeout: 2h privileged: true params: COVERAGE: false DB: postgresql RAKE_TASK: spec:unit:director + PG_VERSION: "17" - name: unit-director-postgres-15 public: true @@ -247,6 +252,7 @@ jobs: COVERAGE: false DB: postgresql RAKE_TASK: spec:unit:director + PG_VERSION: "15" - name: run-bundle-audit public: true @@ -311,8 +317,9 @@ jobs: params: COVERAGE: false DB: postgresql + PG_VERSION: "15" RAKE_TASK: spec:integration - PARALLEL_TEST_MULTIPLY_PROCESSES: "0.5" + PARALLEL_TEST_MULTIPLY_PROCESSES: ¶llel_test_multiplier "0.6" - name: integration-postgres-hotswap public: true @@ -340,8 +347,10 @@ jobs: privileged: true params: DB: postgresql + PG_VERSION: "15" RAKE_TASK: spec:integration UPDATE_VM_STRATEGY: create-swap-delete + PARALLEL_TEST_MULTIPLY_PROCESSES: *parallel_test_multiplier - name: integration-mysql public: true @@ -371,6 +380,7 @@ jobs: COVERAGE: false DB: mysql RAKE_TASK: spec:integration + PARALLEL_TEST_MULTIPLY_PROCESSES: *parallel_test_multiplier - name: candidate-release plan: @@ -385,7 +395,7 @@ jobs: - unit-release - unit-director-sqlite - unit-director-mysql-8 - - unit-director-postgres-13 + - unit-director-postgres-17 - unit-director-postgres-15 - integration-mysql - integration-postgres @@ -674,6 +684,138 @@ jobs: vars: *brats-terraform-vars action: destroy + - name: brats-postgres-15 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-15 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "15" + BRATS_FOCUS_REGEXP: "postgres-release" + + - name: brats-postgres-16 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-16 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "16" + BRATS_FOCUS_REGEXP: "postgres-release" + + - name: brats-postgres-17 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-17 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "17" + BRATS_FOCUS_REGEXP: "postgres-release" + + - name: brats-postgres-18 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-18 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "18" + BRATS_FOCUS_REGEXP: "postgres-release" + - name: brats-performance serial: true plan: @@ -954,6 +1096,10 @@ jobs: passed: - bats - brats-acceptance + - brats-postgres-15 + - brats-postgres-16 + - brats-postgres-17 + - brats-postgres-18 - bosh-disaster-recovery-acceptance-tests - upgrade-postgres - upgrade-mysql @@ -1176,7 +1322,7 @@ jobs: params: image: image/image.tar - - name: build-main-postgres-13 + - name: build-main-postgres-17 public: true serial: true plan: @@ -1200,11 +1346,11 @@ jobs: - name: image params: CONTEXT: bosh-ci-dockerfiles/ci/dockerfiles/main-postgres - BUILD_ARG_DB_VERSION: "13" + BUILD_ARG_DB_VERSION: "17" IMAGE_ARG_BASE_IMAGE: integration-image/image.tar run: path: build - - put: integration-postgres-13-image + - put: integration-postgres-17-image no_get: true params: image: image/image.tar @@ -1959,10 +2105,10 @@ resources: username: ((github_read_write_packages.username)) password: ((github_read_write_packages.password)) - - name: integration-postgres-13-image + - name: integration-postgres-17-image type: registry-image source: - repository: ghcr.io/cloudfoundry/bosh/main-postgres-13 + repository: ghcr.io/cloudfoundry/bosh/main-postgres-17 tag: *branch_name username: ((github_read_write_packages.username)) password: ((github_read_write_packages.password)) diff --git a/src/brats/acceptance/postgres_release_test.go b/src/brats/acceptance/postgres_release_test.go new file mode 100644 index 00000000000..fe47820e575 --- /dev/null +++ b/src/brats/acceptance/postgres_release_test.go @@ -0,0 +1,79 @@ +package acceptance_test + +import ( + "fmt" + "os" + "strconv" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "brats/utils" +) + +var _ = Describe("postgres-release version matrix", func() { + const deployTimeout = 20 * time.Minute + + manifest := utils.AssetPath("postgres-release-manifest.yml") + + type pgEntry struct { + version int + previous int + } + + pgVersionFilter := func() int { + v := os.Getenv("PG_VERSION") + if v == "" { + return 0 + } + n, _ := strconv.Atoi(v) + return n + }() + + DescribeTable("PostgreSQL version", + func(e pgEntry) { + if pgVersionFilter != 0 && e.version != pgVersionFilter { + Skip(fmt.Sprintf("PG_VERSION=%d; skipping version %d", pgVersionFilter, e.version)) + } + deploymentName := fmt.Sprintf("postgres-release-%d-%x", e.version, GinkgoT().RandomSeed()) + + By(fmt.Sprintf("deploying postgres-release at version %d", e.version)) + session := utils.OuterBosh("deploy", "-n", manifest, + "-d", deploymentName, + "-v", fmt.Sprintf("stemcell-os=%s", utils.StemcellOS()), + "-v", fmt.Sprintf("deployment-name=%s", deploymentName), + "-v", fmt.Sprintf("pg-version=%d", e.version), + ) + Eventually(session, deployTimeout).Should(gexec.Exit(0)) + + if e.previous > 0 { + By(fmt.Sprintf("upgrading from postgres-release version %d to %d", e.previous, e.version)) + previousDeploymentName := fmt.Sprintf("postgres-release-%d-to-%d-%x", e.previous, e.version, GinkgoT().RandomSeed()) + + By(fmt.Sprintf("deploying at version %d first", e.previous)) + session = utils.OuterBosh("deploy", "-n", manifest, + "-d", previousDeploymentName, + "-v", fmt.Sprintf("stemcell-os=%s", utils.StemcellOS()), + "-v", fmt.Sprintf("deployment-name=%s", previousDeploymentName), + "-v", fmt.Sprintf("pg-version=%d", e.previous), + ) + Eventually(session, deployTimeout).Should(gexec.Exit(0)) + + By(fmt.Sprintf("upgrading to version %d", e.version)) + session = utils.OuterBosh("deploy", "-n", manifest, + "-d", previousDeploymentName, + "-v", fmt.Sprintf("stemcell-os=%s", utils.StemcellOS()), + "-v", fmt.Sprintf("deployment-name=%s", previousDeploymentName), + "-v", fmt.Sprintf("pg-version=%d", e.version), + ) + Eventually(session, deployTimeout).Should(gexec.Exit(0)) + } + }, + Entry("deploys version 15", pgEntry{version: 15}), + Entry("upgrades 15 -> 16", pgEntry{version: 16, previous: 15}), + Entry("upgrades 16 -> 17", pgEntry{version: 17, previous: 16}), + Entry("upgrades 17 -> 18", pgEntry{version: 18, previous: 17}), + ) +}) diff --git a/src/brats/assets/postgres-13-manifest.yml b/src/brats/assets/postgres-13-manifest.yml index fd132f9dc61..54d3615c2a8 100644 --- a/src/brats/assets/postgres-13-manifest.yml +++ b/src/brats/assets/postgres-13-manifest.yml @@ -6,7 +6,7 @@ instance_groups: instances: 1 jobs: - name: postgres - release: bosh + release: postgres - name: bpm release: bpm vm_type: default @@ -15,10 +15,14 @@ instance_groups: networks: - name: default properties: - postgres: - user: postgres - password: c1oudc0w - database: bosh + databases: + version: 13 + port: 5432 + databases: + - name: bosh + roles: + - name: postgres + password: c1oudc0w stemcells: - alias: default @@ -26,10 +30,12 @@ stemcells: version: latest releases: -- name: "bosh" +- name: bosh version: "276.1.1" url: "https://bosh.io/d/github.com/cloudfoundry/bosh?v=276.1.1" sha1: "f9a625dd8a8fc6e01f1641390ced3ac0fee31523" +- name: postgres + version: latest - name: bpm version: latest diff --git a/src/brats/assets/postgres-manifest.yml b/src/brats/assets/postgres-manifest.yml index 6181a8a6f00..90c0ca73120 100644 --- a/src/brats/assets/postgres-manifest.yml +++ b/src/brats/assets/postgres-manifest.yml @@ -6,7 +6,7 @@ instance_groups: instances: 1 jobs: - name: postgres - release: bosh + release: postgres - name: bpm release: bpm vm_type: default @@ -15,13 +15,14 @@ instance_groups: networks: - name: default properties: - postgres: - listen_address: 127.0.0.1 - host: 127.0.0.1 - user: postgres - password: c1oudc0w - database: bosh - adapter: postgres + databases: + version: 15 + port: 5432 + databases: + - name: bosh + roles: + - name: postgres + password: c1oudc0w stemcells: - alias: default @@ -31,6 +32,8 @@ stemcells: releases: - name: bosh version: latest +- name: postgres + version: latest - name: bpm version: latest diff --git a/src/brats/assets/postgres-release-manifest.yml b/src/brats/assets/postgres-release-manifest.yml new file mode 100644 index 00000000000..885ed2b54d1 --- /dev/null +++ b/src/brats/assets/postgres-release-manifest.yml @@ -0,0 +1,44 @@ +--- +name: ((deployment-name)) +instance_groups: +- name: bosh + azs: [z1] + instances: 1 + jobs: + - name: postgres + release: postgres + - name: bpm + release: bpm + vm_type: default + stemcell: default + persistent_disk_type: default + networks: + - name: default + properties: + databases: + version: ((pg-version)) + port: 5432 + databases: + - name: bosh + roles: + - name: postgres + password: c1oudc0w + +stemcells: +- alias: default + os: ((stemcell-os)) + version: latest + +releases: +- name: bosh + version: latest +- name: postgres + version: latest +- name: bpm + version: latest + +update: + canaries: 1 + max_in_flight: 10 + canary_watch_time: 1000-30000 + update_watch_time: 1000-30000 diff --git a/src/spec/integration_support/postgres_version_helper.rb b/src/spec/integration_support/postgres_version_helper.rb index 4603305548a..cbb4842c1ee 100644 --- a/src/spec/integration_support/postgres_version_helper.rb +++ b/src/spec/integration_support/postgres_version_helper.rb @@ -1,4 +1,3 @@ -require 'yaml' require 'integration_support/constants' module IntegrationSupport @@ -7,8 +6,8 @@ class << self def ensure_version_match!(env_db) return unless env_db == 'postgresql' - unless local_major_version == release_major_version - raise "Postgres major version mismatch: jobs/postgres/spec: #{local_version}; local: #{release_version}." + unless local_major_version == configured_major_version + raise "Postgres major version mismatch: PG_VERSION=#{configured_version}; local: #{local_version}." end end @@ -20,29 +19,20 @@ def local_version `postgres --version`.chomp.split(' ').last end - def release_major_version - release_version.split('.')[0] + def configured_major_version + configured_version.split('.')[0] end - def release_version - @release_version ||= begin - postgres_release_config = - YAML.load_file( - File.join(IntegrationSupport::Constants::BOSH_REPO_ROOT, 'jobs', 'postgres', 'spec'), - permitted_classes: [Symbol], - aliases: true, - ) - - # sort alphanumerics correctly, e.g. 10 > 9 - postgres_version = postgres_release_config['packages'].max_by { |s| s.scan(/\d+/).first.to_i } - - postgres_version.split('-').last + def configured_version + ENV.fetch('PG_VERSION') do + raise 'PG_VERSION environment variable must be set when DB=postgresql' end end end end end + RSpec.configure do |c| c.before(:suite) do IntegrationSupport::PostgresVersionHelper.ensure_version_match!(ENV['DB'])