diff --git a/.github/workflows/create-scratch-utils-tag.yml b/.github/workflows/create-scratch-utils-tag.yml new file mode 100644 index 00000000..cee2ff2d --- /dev/null +++ b/.github/workflows/create-scratch-utils-tag.yml @@ -0,0 +1,49 @@ +name: Create new Git tag on scratch-utils version change + +on: + # Tag a scratch-utils release when its version lands on master. The publish + # promotion fires on that tag, so an image tag cannot be republished with new + # contents -- doing so needs a new version here, which makes a new tag. + push: + branches: + - master + paths: + - images/calico-scratch-utils/versions.yaml + +jobs: + create-tag: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Generate tag name + id: generate-tag-name + run: | + version=$(yq -r '.scratch-utils.version' images/calico-scratch-utils/versions.yaml) + if [ -z "$version" ] || [ "$version" = "null" ]; then + echo "scratch-utils.version is empty" >&2 + exit 1 + fi + # Prefixed: this repo's release tags live in the same namespace as + # 1.27.1-llvm21.1.8-k8s1.37.0, and a second binary here would need its own. + tag_name="scratch-utils-${version}" + echo "Git tag name: $tag_name" + echo "tag_name=$tag_name" >> $GITHUB_OUTPUT + + - name: Create and push new tag (if not exists) + run: | + tag_name=${{ steps.generate-tag-name.outputs.tag_name }} + if git ls-remote --tags --refs origin "$tag_name" | grep -q "$tag_name"; then + echo "Git tag $tag_name already exists" + exit 0 + fi + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git + git tag -a "$tag_name" -m "Release $tag_name" + git push origin "$tag_name" + echo "Created Git tag $tag_name" diff --git a/.gitignore b/.gitignore index bac9bc32..2a808d15 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ cmd/bin images/calico-binfmt/bin images/calico-go-build/bin +images/calico-scratch-utils/bin diff --git a/.semaphore/promotions/calico-base.yml b/.semaphore/promotions/calico-base.yml index 5a12b8ab..708ee16d 100644 --- a/.semaphore/promotions/calico-base.yml +++ b/.semaphore/promotions/calico-base.yml @@ -23,7 +23,7 @@ blocks: - name: Publish calico/base multi-arch images dependencies: [] run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" task: env_vars: - name: BRANCH_NAME diff --git a/.semaphore/promotions/calico-binfmt.yml b/.semaphore/promotions/calico-binfmt.yml index 846d94bf..d1da08f3 100644 --- a/.semaphore/promotions/calico-binfmt.yml +++ b/.semaphore/promotions/calico-binfmt.yml @@ -23,7 +23,7 @@ blocks: - name: Publish calico/binfmt amd64 images dependencies: [] run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" task: jobs: - name: Linux amd64 diff --git a/.semaphore/promotions/calico-go-build.yml b/.semaphore/promotions/calico-go-build.yml index d0c31d8d..f65c8870 100644 --- a/.semaphore/promotions/calico-go-build.yml +++ b/.semaphore/promotions/calico-go-build.yml @@ -28,11 +28,24 @@ global_job_config: export BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} fi +# Both images cache or bake calico/go-build, so they are rebuilt once it is +# published -- chained here rather than off the root pipeline so they cannot start +# before the image they consume exists. +promotions: + - name: Build ci-base VM image + pipeline_file: ci-base-vm-image.yml + auto_promote: + when: "result = 'passed' AND tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" + - name: Build go-build preload disk image + pipeline_file: preload-disk-image.yml + auto_promote: + when: "result = 'passed' AND tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" + blocks: - name: Publish calico/go-build images dependencies: [] run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" task: jobs: - name: Linux multi-arch @@ -45,7 +58,7 @@ blocks: dependencies: - Publish calico/go-build images run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" task: jobs: - name: Linux multi-arch manifests diff --git a/.semaphore/promotions/calico-scratch-utils.yml b/.semaphore/promotions/calico-scratch-utils.yml new file mode 100644 index 00000000..b37317ec --- /dev/null +++ b/.semaphore/promotions/calico-scratch-utils.yml @@ -0,0 +1,37 @@ +version: v1.0 +name: Publish calico/scratch-utils images +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 + +execution_time_limit: + minutes: 30 + +global_job_config: + env_vars: + - name: DEV_REGISTRIES + value: calico + secrets: + - name: docker + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - checkout + - | + if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then + export BRANCH_NAME=${SEMAPHORE_GIT_TAG_NAME#scratch-utils-} + else + export BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} + fi + +blocks: + - name: Publish calico/scratch-utils amd64 images + dependencies: [] + run: + when: "branch = 'master' OR tag =~ '^scratch-utils-v'" + task: + jobs: + - name: Linux amd64 + commands: + - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-scratch-utils-cd CONFIRM=true; fi diff --git a/.semaphore/promotions/ci-base-vm-image.yml b/.semaphore/promotions/ci-base-vm-image.yml new file mode 100644 index 00000000..300beae6 --- /dev/null +++ b/.semaphore/promotions/ci-base-vm-image.yml @@ -0,0 +1,39 @@ +version: v1.0 +name: Build ci-base VM image +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 + +execution_time_limit: + minutes: 30 + +global_job_config: + env_vars: + - name: PROJECT + value: unique-caldron-775 + - name: ZONE + value: us-central1-a + - name: FAMILY + value: ci-base + secrets: + - name: google-service-account-for-gce + prologue: + commands: + - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json + - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS + - gcloud config set project $PROJECT + - checkout + # build-image.sh reads the versions.yaml files with yq. + - which yq || (sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && sudo chmod +x /usr/local/bin/yq) + +blocks: + - name: Build ci-base VM image + dependencies: [] + run: + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" + task: + jobs: + - name: build-image + commands: + - bash vm-images/ci-base/build-image.sh diff --git a/.semaphore/promotions/preload-disk-image.yml b/.semaphore/promotions/preload-disk-image.yml new file mode 100644 index 00000000..35fab142 --- /dev/null +++ b/.semaphore/promotions/preload-disk-image.yml @@ -0,0 +1,41 @@ +version: v1.0 +name: Build go-build preload disk image +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 + +execution_time_limit: + minutes: 30 + +global_job_config: + env_vars: + - name: PROJECT + value: tigera-cc-dev + - name: ZONE + value: us-central1-a + - name: GCS_PATH + value: gs://gke-argo-disk-images-cc-dev + secrets: + - name: google-service-account-for-gce + prologue: + commands: + - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json + - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS + - gcloud config set project $PROJECT + # gke-disk-image-builder is `go run ./cli`; pick the Go the tool's go.mod needs. + - sem-version go 1.24 + - checkout + # build-preload-disk.sh resolves the go-build tag from versions.yaml with yq. + - which yq || (sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && sudo chmod +x /usr/local/bin/yq) + +blocks: + - name: Build preload disk image + dependencies: [] + run: + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" + task: + jobs: + - name: build-preload-disk + commands: + - bash preload-disks/ci-cache/build-preload-disk.sh diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a4a7db92..f3bbdf81 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -29,15 +29,15 @@ promotions: - name: Publish calico/base images pipeline_file: promotions/calico-base.yml auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" - name: Publish calico/binfmt images pipeline_file: promotions/calico-binfmt.yml auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" - name: Publish calico/go-build images pipeline_file: promotions/calico-go-build.yml auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" + when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+(\\.\\d+){2}-k8s1\\.\\d+\\.\\d+'" # Publish rust-build images for master and files under images/calico-rust-build have changed. - name: Publish calico/rust-build images pipeline_file: promotions/calico-rust-build.yml @@ -48,6 +48,13 @@ promotions: pipeline_file: promotions/calico-tinygo.yml auto_promote: when: "branch = 'master' AND change_in('/images/calico-tinygo/')" + # master publishes a moving :master, so any change ships -- including a bare + # go.mod bump, which no change_in path would have caught. A scratch-utils-v* tag + # publishes the immutable :vX.Y that workflows pin. + - name: Publish calico/scratch-utils images + pipeline_file: promotions/calico-scratch-utils.yml + auto_promote: + when: "branch = 'master' OR tag =~ '^scratch-utils-v'" blocks: - name: calico/go-build image @@ -186,3 +193,22 @@ blocks: matrix: - env_var: ARCH values: ["amd64", "arm64"] + + - name: calico/scratch-utils image + dependencies: [] + task: + jobs: + - name: Build calico/scratch-utils amd64 image + commands: + - make -C images calico-scratch-utils-image ARCH=amd64 + + # Run in the golang image at the version versions.yaml pins, so the tests see the + # same Go the go-build image ships and the agent needs none of its own. + - name: Go unit tests + dependencies: [] + task: + jobs: + - name: go test + commands: + - GO_VERSION=$(hack/generate-version-tag-name.sh -f images/calico-go-build/versions.yaml -g) + - docker run --rm -v "$PWD:/src" -w /src golang:${GO_VERSION} go test ./scratch-utils/... ./cmd/... diff --git a/cmd/Makefile b/cmd/Makefile index 5dc102a4..ff78bc69 100644 --- a/cmd/Makefile +++ b/cmd/Makefile @@ -9,6 +9,9 @@ QEMU_VERSION = $(shell yq -r '.qemu.version' ../images/calico-binfmt/versions.ya .PHONY: build build: bin/binfmt-$(ARCH) bin/semvalidator-$(ARCH) +.PHONY: scratch-utils-build +scratch-utils-build: bin/scratch-utils-$(ARCH) + bin/binfmt-$(ARCH): binfmt/main.go CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) \ go build -o bin/binfmt-$(ARCH) -v -buildvcs=false -ldflags "-X main.revision=$(BINFMT_VERSION) -X main.qemuVersion=$(QEMU_VERSION) -s -w" binfmt/*.go @@ -17,6 +20,15 @@ bin/semvalidator-$(ARCH): semvalidator/main.go CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) \ go build -o bin/semvalidator-$(ARCH) -v -buildvcs=false -ldflags "-s -w" semvalidator/main.go +# Unlike the single-file binaries above, this one spans ../scratch-utils/, so list +# every .go file it compiles -- otherwise an incremental build ships a stale binary. +# := so the find runs once, not on every expansion of the prerequisite list. +SCRATCH_UTILS_SRCS := $(shell find scratch-utils ../scratch-utils -name '*.go') + +bin/scratch-utils-$(ARCH): $(SCRATCH_UTILS_SRCS) + CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) \ + go build -o bin/scratch-utils-$(ARCH) -v -buildvcs=false -ldflags "-s -w" ./scratch-utils + .PHONY: clean clean: rm -fr bin/ diff --git a/cmd/scratch-utils/main.go b/cmd/scratch-utils/main.go new file mode 100644 index 00000000..70e41db3 --- /dev/null +++ b/cmd/scratch-utils/main.go @@ -0,0 +1,53 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Command scratch-utils bundles the CI VM helpers into one binary dispatched by +// subcommand -- one image, different args, not four binaries: +// +// scratch-utils createvm create the CI GCE VM (config from env) +// scratch-utils deletevm delete it by name, best-effort +// scratch-utils secret env var -> file +// scratch-utils runonvm [flags]