diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7d7b8b..f5cf20c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,6 +26,14 @@ env: # renovate: datasource=github-releases depName=helm-unittest/helm-unittest HELM_UNITTEST_VERSION: "v1.1.2" + # The tag e2e-image builds under and every leg pushes to its own runner-local + # registry. The nodes pull the registry.e2e name instead (see the go-e2e env), + # which the Talos mirror in patches/registry.yaml points at port 5000 on the + # bridge gateway. The split matters beyond taste: test.sh splits + # CONTROLLER_IMAGE on ':' for the Helm repo and tag, so the host:port push form + # would parse wrong there. + PUSH_IMAGE: localhost:5000/tuppr:e2e + jobs: docs: if: ${{ github.event_name != 'push' }} @@ -36,12 +44,74 @@ jobs: with: deploy: false + # Built once and handed to every e2e leg, so they all install identical bytes + # instead of four builds racing one GHA cache scope. + e2e-image: + if: >- + ${{ github.event_name != 'push' + && !(startsWith(github.head_ref, 'release-please--') + && github.event.pull_request.head.repo.full_name == github.repository) }} + name: E2E Image + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Nothing here runs a pinned tool; the build happens inside Docker and only + # the Go version is read back out of the config. + - name: Setup Mise + uses: jdx/mise-action@9e7f7633ff6f6d6048a9418a68d48f288f50eb14 # v4.2.3 + with: + install: false + + - name: Resolve toolchain versions + id: tools + run: echo "go=$(mise config get tools.go)" >> "$GITHUB_OUTPUT" + + - name: Set up Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # load: true rather than a push: this job owns no registry, and one docker + # save is what the legs get handed. One writer per scope now, so cache-to + # is unconditional. + - name: Build controller image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + load: true + tags: ${{ env.PUSH_IMAGE }} + build-args: GO_VERSION=${{ steps.tools.outputs.go }} + cache-from: type=gha,scope=e2e-controller + cache-to: type=gha,mode=max,scope=e2e-controller + + # gzip because the daemon may export layers uncompressed depending on its + # image store, and docker load reads either. shell: bash for the pipefail + # the implicit `bash -e` default lacks -- a failed save would otherwise + # leave gzip's exit status to pass the step and hand every leg a truncated + # tarball. + - name: Export the image + shell: bash + run: docker save "${PUSH_IMAGE}" | gzip -1 > "${RUNNER_TEMP}/e2e-image.tar.gz" + + - name: Upload the image + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-image + path: ${{ runner.temp }}/e2e-image.tar.gz + retention-days: 1 + go-e2e: if: >- ${{ github.event_name != 'push' && !(startsWith(github.head_ref, 'release-please--') && github.event.pull_request.head.repo.full_name == github.repository) }} name: Go E2E (${{ matrix.upgrade }} ${{ matrix.leg }}) + needs: e2e-image runs-on: ubuntu-24.04 permissions: contents: read @@ -75,13 +145,10 @@ jobs: - leg: 1cp-1w upgrade: kubernetes env: - # Two names for one image in the runner-local registry. The nodes pull - # registry.e2e, which the Talos mirror in patches/registry.yaml points at - # the bridge gateway; buildx pushes to the same registry over loopback. - # The alias matters beyond taste: test.sh splits CONTROLLER_IMAGE on ':' - # to get the Helm repo and tag, so a host:port form would parse wrong. + # What the chart is pointed at, and so what the nodes pull: the other half + # of the two-names-for-one-image split described on the PUSH_IMAGE var + # above. CONTROLLER_IMAGE: registry.e2e/tuppr:e2e - PUSH_IMAGE: localhost:5000/tuppr:e2e # test.sh packages the chart, pushes it here, and installs it back over # OCI, so the run exercises the distributed artifact rather than the # working tree. Same runner-local registry as the image, over loopback, so @@ -118,17 +185,11 @@ jobs: sudo mkswap /mnt/e2e-swapfile sudo swapon /mnt/e2e-swapfile - - name: Resolve toolchain versions - id: tools - working-directory: . - run: echo "go=$(mise config get tools.go)" >> "$GITHUB_OUTPUT" - - name: Start image registry run: docker run -d --name e2e-registry -p 5000:5000 registry:3 - # Booting the VMs and building the controller share no inputs, so they - # overlap. They do contend for the runner's 4 vCPUs, so the saving is - # less than the build time, but the build leaves the critical path. + # Booting the VMs and filling the registry share no inputs, so they overlap + # the way the build this leg used to run did. # # The config path is relative to the workspace: defaults.run applies to # run steps only. talosctl comes from mise, whose shim the action unwraps @@ -140,29 +201,20 @@ jobs: with: config: test/e2e-qemu/${{ matrix.leg }}.yaml - # network=host so the builder container can reach the registry published - # on the runner's loopback. - - name: Set up Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - name: Download the image + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - driver-opts: network=host + name: e2e-image + path: ${{ runner.temp }} - # Every leg builds a byte-identical image, so they share one cache scope. - # Only one writes it back: concurrent cache-to writes to the same scope - # evict each other. 3cp-0w is unique across the matrix, so it names one leg. - - name: Build controller image - id: image - background: true - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - push: true - tags: ${{ env.PUSH_IMAGE }} - build-args: GO_VERSION=${{ steps.tools.outputs.go }} - cache-from: type=gha,scope=e2e-controller - cache-to: ${{ matrix.leg == '3cp-0w' && 'type=gha,mode=max,scope=e2e-controller' || '' }} + # Straight into this leg's own registry, so the nodes still pull over + # loopback and nothing crosses the internet. + - name: Load the image into the registry + run: | + docker load -i "${RUNNER_TEMP}/e2e-image.tar.gz" + docker push "${PUSH_IMAGE}" - - wait: [cluster, image] + - wait: [cluster] # The action exports KUBECONFIG and TALOSCONFIG too, but naming the outputs # keeps it visible which cluster the tests are pointed at, and test.sh @@ -342,6 +394,7 @@ jobs: name: Build Success needs: - docs + - e2e-image - go-e2e - go-lint - go-test diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 3d0c032..6e53c2b 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,16 +1,13 @@ --- rules: - # The e2e legs (the go-e2e job in ci.yaml) build an image with `load: true`: - # it is loaded into the kind cluster for the test and never pushed to a - # registry or released, so there is no published artifact for a poisoned - # tool/layer cache to compromise. - # The e2e workflow builds and pushes a controller image in a job where - # mise-action has caching on, which the rule reads as a cached tool feeding a - # published artifact. It does not apply here on two counts: the Dockerfile - # builds FROM the official golang image, so no mise-managed tool reaches the - # binary (mise supplies only the version string), and the image is pushed to a - # registry running on the runner itself, so it never leaves the job that built - # it. + # The e2e-image job in ci.yaml builds a controller image from a cached layer + # set, in a job where mise-action has caching on, which the rule reads as a + # cached tool feeding a published artifact. It does not apply here on two + # counts: the Dockerfile builds FROM the official golang image, so no + # mise-managed tool reaches the binary (mise supplies only the version + # string), and what it produces only ever reaches the throwaway QEMU clusters + # the e2e legs boot -- a runner-local registry, never one outside it and never + # a release. cache-poisoning: ignore: - ci.yaml diff --git a/test/e2e-qemu/README.md b/test/e2e-qemu/README.md index 889c765..050ace5 100644 --- a/test/e2e-qemu/README.md +++ b/test/e2e-qemu/README.md @@ -84,14 +84,17 @@ When `CONTROLLER_IMAGE` is unset, `test.sh` builds and pushes one itself via set -gx CONTROLLER_IMAGE ghcr.io/you/tuppr:dev ``` -CI instead fixes the tag up front and builds it concurrently with the cluster, -since the build and the VMs share no inputs; `test.sh` then finds the image already -built and skips straight to installing it. CI builds through -`docker/build-push-action` rather than `image.sh` so the layers land in the GitHub -Actions cache, which `image.sh` has no way to reach. - -CI also runs its own registry on the runner and pushes there, so the image never -crosses the internet. The nodes reach it as `registry.e2e`, which the mirror in +CI instead builds it once, in the `e2e-image` job that runs ahead of the matrix, +and hands it to every leg as a `docker save` tarball. Each leg loads that tarball +while its VMs boot and pushes it into its own registry, so the legs install +identical bytes rather than whatever each rebuild produced, and one job owns the +GitHub Actions cache scope instead of every leg racing for it. `test.sh` finds the +image already tagged and skips straight to installing it. CI builds through +`docker/build-push-action` rather than `image.sh` so the layers land in that cache, +which `image.sh` has no way to reach. + +Each leg still runs its own registry on the runner, so the image never crosses the +internet. The nodes reach it as `registry.e2e`, which the mirror in `patches/registry.yaml` points at port 5000 on the QEMU bridge gateway. That mirror entry is inert for a local run, where nothing references `registry.e2e`. diff --git a/test/e2e-qemu/test.sh b/test/e2e-qemu/test.sh index 20e8b50..43b50ff 100755 --- a/test/e2e-qemu/test.sh +++ b/test/e2e-qemu/test.sh @@ -220,8 +220,8 @@ main() { cd "$REPO_ROOT" - # CI sets CONTROLLER_IMAGE and builds it concurrently with the cluster, so - # by the time we get here there is nothing to do. + # CI sets CONTROLLER_IMAGE and loads a prebuilt one into its registry while + # the cluster boots, so by the time we get here there is nothing to do. if [[ -n "${CONTROLLER_IMAGE:-}" ]]; then log "Using pre-built controller image: $CONTROLLER_IMAGE" else