From ce1b2a2b241742fb68ac099256d3bc965651a32c Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 29 Jul 2026 13:45:54 -0400 Subject: [PATCH] ci: build the e2e image once and hand it to every leg Every leg rebuilt the controller image itself, which meant four builds racing one GHA cache scope and an ordering hack where only the 3cp-0w leg wrote the scope back. A new e2e-image job builds it once, exports it with docker save, and hands it to the matrix as an artifact. Each leg loads it into its own runner-local registry while its VMs boot, so the nodes still pull over loopback and every leg now installs identical bytes. Export uses shell: bash for the pipefail the implicit `bash -e` default lacks, or a failed save would pass the step on gzip's exit status and hand every leg a truncated tarball. Signed-off-by: Devin Buhl --- .github/workflows/ci.yaml | 121 +++++++++++++++++++++++++++----------- .github/zizmor.yml | 19 +++--- test/e2e-qemu/README.md | 19 +++--- test/e2e-qemu/test.sh | 4 +- 4 files changed, 108 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7d7b8b..f5cf20c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,6 +26,14 @@ env: # renovate: datasource=github-releases depName=helm-unittest/helm-unittest HELM_UNITTEST_VERSION: "v1.1.2" + # The tag e2e-image builds under and every leg pushes to its own runner-local + # registry. The nodes pull the registry.e2e name instead (see the go-e2e env), + # which the Talos mirror in patches/registry.yaml points at port 5000 on the + # bridge gateway. The split matters beyond taste: test.sh splits + # CONTROLLER_IMAGE on ':' for the Helm repo and tag, so the host:port push form + # would parse wrong there. + PUSH_IMAGE: localhost:5000/tuppr:e2e + jobs: docs: if: ${{ github.event_name != 'push' }} @@ -36,12 +44,74 @@ jobs: with: deploy: false + # Built once and handed to every e2e leg, so they all install identical bytes + # instead of four builds racing one GHA cache scope. + e2e-image: + if: >- + ${{ github.event_name != 'push' + && !(startsWith(github.head_ref, 'release-please--') + && github.event.pull_request.head.repo.full_name == github.repository) }} + name: E2E Image + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Nothing here runs a pinned tool; the build happens inside Docker and only + # the Go version is read back out of the config. + - name: Setup Mise + uses: jdx/mise-action@9e7f7633ff6f6d6048a9418a68d48f288f50eb14 # v4.2.3 + with: + install: false + + - name: Resolve toolchain versions + id: tools + run: echo "go=$(mise config get tools.go)" >> "$GITHUB_OUTPUT" + + - name: Set up Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # load: true rather than a push: this job owns no registry, and one docker + # save is what the legs get handed. One writer per scope now, so cache-to + # is unconditional. + - name: Build controller image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + load: true + tags: ${{ env.PUSH_IMAGE }} + build-args: GO_VERSION=${{ steps.tools.outputs.go }} + cache-from: type=gha,scope=e2e-controller + cache-to: type=gha,mode=max,scope=e2e-controller + + # gzip because the daemon may export layers uncompressed depending on its + # image store, and docker load reads either. shell: bash for the pipefail + # the implicit `bash -e` default lacks -- a failed save would otherwise + # leave gzip's exit status to pass the step and hand every leg a truncated + # tarball. + - name: Export the image + shell: bash + run: docker save "${PUSH_IMAGE}" | gzip -1 > "${RUNNER_TEMP}/e2e-image.tar.gz" + + - name: Upload the image + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-image + path: ${{ runner.temp }}/e2e-image.tar.gz + retention-days: 1 + go-e2e: if: >- ${{ github.event_name != 'push' && !(startsWith(github.head_ref, 'release-please--') && github.event.pull_request.head.repo.full_name == github.repository) }} name: Go E2E (${{ matrix.upgrade }} ${{ matrix.leg }}) + needs: e2e-image runs-on: ubuntu-24.04 permissions: contents: read @@ -75,13 +145,10 @@ jobs: - leg: 1cp-1w upgrade: kubernetes env: - # Two names for one image in the runner-local registry. The nodes pull - # registry.e2e, which the Talos mirror in patches/registry.yaml points at - # the bridge gateway; buildx pushes to the same registry over loopback. - # The alias matters beyond taste: test.sh splits CONTROLLER_IMAGE on ':' - # to get the Helm repo and tag, so a host:port form would parse wrong. + # What the chart is pointed at, and so what the nodes pull: the other half + # of the two-names-for-one-image split described on the PUSH_IMAGE var + # above. CONTROLLER_IMAGE: registry.e2e/tuppr:e2e - PUSH_IMAGE: localhost:5000/tuppr:e2e # test.sh packages the chart, pushes it here, and installs it back over # OCI, so the run exercises the distributed artifact rather than the # working tree. Same runner-local registry as the image, over loopback, so @@ -118,17 +185,11 @@ jobs: sudo mkswap /mnt/e2e-swapfile sudo swapon /mnt/e2e-swapfile - - name: Resolve toolchain versions - id: tools - working-directory: . - run: echo "go=$(mise config get tools.go)" >> "$GITHUB_OUTPUT" - - name: Start image registry run: docker run -d --name e2e-registry -p 5000:5000 registry:3 - # Booting the VMs and building the controller share no inputs, so they - # overlap. They do contend for the runner's 4 vCPUs, so the saving is - # less than the build time, but the build leaves the critical path. + # Booting the VMs and filling the registry share no inputs, so they overlap + # the way the build this leg used to run did. # # The config path is relative to the workspace: defaults.run applies to # run steps only. talosctl comes from mise, whose shim the action unwraps @@ -140,29 +201,20 @@ jobs: with: config: test/e2e-qemu/${{ matrix.leg }}.yaml - # network=host so the builder container can reach the registry published - # on the runner's loopback. - - name: Set up Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - name: Download the image + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - driver-opts: network=host + name: e2e-image + path: ${{ runner.temp }} - # Every leg builds a byte-identical image, so they share one cache scope. - # Only one writes it back: concurrent cache-to writes to the same scope - # evict each other. 3cp-0w is unique across the matrix, so it names one leg. - - name: Build controller image - id: image - background: true - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - push: true - tags: ${{ env.PUSH_IMAGE }} - build-args: GO_VERSION=${{ steps.tools.outputs.go }} - cache-from: type=gha,scope=e2e-controller - cache-to: ${{ matrix.leg == '3cp-0w' && 'type=gha,mode=max,scope=e2e-controller' || '' }} + # Straight into this leg's own registry, so the nodes still pull over + # loopback and nothing crosses the internet. + - name: Load the image into the registry + run: | + docker load -i "${RUNNER_TEMP}/e2e-image.tar.gz" + docker push "${PUSH_IMAGE}" - - wait: [cluster, image] + - wait: [cluster] # The action exports KUBECONFIG and TALOSCONFIG too, but naming the outputs # keeps it visible which cluster the tests are pointed at, and test.sh @@ -342,6 +394,7 @@ jobs: name: Build Success needs: - docs + - e2e-image - go-e2e - go-lint - go-test diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 3d0c032..6e53c2b 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,16 +1,13 @@ --- rules: - # The e2e legs (the go-e2e job in ci.yaml) build an image with `load: true`: - # it is loaded into the kind cluster for the test and never pushed to a - # registry or released, so there is no published artifact for a poisoned - # tool/layer cache to compromise. - # The e2e workflow builds and pushes a controller image in a job where - # mise-action has caching on, which the rule reads as a cached tool feeding a - # published artifact. It does not apply here on two counts: the Dockerfile - # builds FROM the official golang image, so no mise-managed tool reaches the - # binary (mise supplies only the version string), and the image is pushed to a - # registry running on the runner itself, so it never leaves the job that built - # it. + # The e2e-image job in ci.yaml builds a controller image from a cached layer + # set, in a job where mise-action has caching on, which the rule reads as a + # cached tool feeding a published artifact. It does not apply here on two + # counts: the Dockerfile builds FROM the official golang image, so no + # mise-managed tool reaches the binary (mise supplies only the version + # string), and what it produces only ever reaches the throwaway QEMU clusters + # the e2e legs boot -- a runner-local registry, never one outside it and never + # a release. cache-poisoning: ignore: - ci.yaml diff --git a/test/e2e-qemu/README.md b/test/e2e-qemu/README.md index 889c765..050ace5 100644 --- a/test/e2e-qemu/README.md +++ b/test/e2e-qemu/README.md @@ -84,14 +84,17 @@ When `CONTROLLER_IMAGE` is unset, `test.sh` builds and pushes one itself via set -gx CONTROLLER_IMAGE ghcr.io/you/tuppr:dev ``` -CI instead fixes the tag up front and builds it concurrently with the cluster, -since the build and the VMs share no inputs; `test.sh` then finds the image already -built and skips straight to installing it. CI builds through -`docker/build-push-action` rather than `image.sh` so the layers land in the GitHub -Actions cache, which `image.sh` has no way to reach. - -CI also runs its own registry on the runner and pushes there, so the image never -crosses the internet. The nodes reach it as `registry.e2e`, which the mirror in +CI instead builds it once, in the `e2e-image` job that runs ahead of the matrix, +and hands it to every leg as a `docker save` tarball. Each leg loads that tarball +while its VMs boot and pushes it into its own registry, so the legs install +identical bytes rather than whatever each rebuild produced, and one job owns the +GitHub Actions cache scope instead of every leg racing for it. `test.sh` finds the +image already tagged and skips straight to installing it. CI builds through +`docker/build-push-action` rather than `image.sh` so the layers land in that cache, +which `image.sh` has no way to reach. + +Each leg still runs its own registry on the runner, so the image never crosses the +internet. The nodes reach it as `registry.e2e`, which the mirror in `patches/registry.yaml` points at port 5000 on the QEMU bridge gateway. That mirror entry is inert for a local run, where nothing references `registry.e2e`. diff --git a/test/e2e-qemu/test.sh b/test/e2e-qemu/test.sh index 20e8b50..43b50ff 100755 --- a/test/e2e-qemu/test.sh +++ b/test/e2e-qemu/test.sh @@ -220,8 +220,8 @@ main() { cd "$REPO_ROOT" - # CI sets CONTROLLER_IMAGE and builds it concurrently with the cluster, so - # by the time we get here there is nothing to do. + # CI sets CONTROLLER_IMAGE and loads a prebuilt one into its registry while + # the cluster boots, so by the time we get here there is nothing to do. if [[ -n "${CONTROLLER_IMAGE:-}" ]]; then log "Using pre-built controller image: $CONTROLLER_IMAGE" else