From d90f9d0878966781c337004a7f3d9b7ea04d24bf Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Tue, 1 Sep 2026 20:49:53 -0700 Subject: [PATCH 01/33] Add cctools CI-tools image + ci-base VM builder cctools is a single static binary (createvm/deletevm/secret/runonvm subcommands) that drives the ArgoCI kind-rig's GCE VM lifecycle over the GCP APIs -- no gcloud, no bash -- so it runs from a distroless image. It is folded into this module (rather than a separate one) since it is CI VM tooling that lives alongside go-build; that adds google.golang.org/api (compute) + golang.org/x/crypto (ssh). - cmd/cctools dispatches the four subcommands; cctools/ holds the shared gce (VM create/delete + native x/crypto/ssh) and ccutil (secret materialization + compute ADC) packages plus the subcommands. - images/calico-cctools: distroless image (carries the CA certs GCP API TLS needs), built + published as calico/cctools via images/Makefile calico-cctools-image/-cd and a Semaphore build block + change-gated promotion. - vm-image: the ci-base GCE image builder (provision.sh bakes docker/go/kind/ kubectl/gh and pre-pulls the heavy images; build-image.sh snapshots it into an image family). Manual for now; a master/release promotion can build it on merge. --- .gitignore | 1 + .semaphore/promotions/calico-cctools.yml | 31 +++ .semaphore/semaphore.yml | 13 + cctools/ccutil/secret.go | 71 +++++ cctools/gce/ssh.go | 332 +++++++++++++++++++++++ cctools/gce/vm.go | 167 ++++++++++++ cctools/subcmd/createvm/createvm.go | 115 ++++++++ cctools/subcmd/deletevm/deletevm.go | 73 +++++ cctools/subcmd/runonvm/runonvm.go | 234 ++++++++++++++++ cctools/subcmd/secret/secret.go | 31 +++ cmd/Makefile | 7 +- cmd/cctools/main.go | 52 ++++ go.mod | 26 ++ go.sum | 67 +++++ images/Makefile | 24 +- images/calico-cctools/Dockerfile | 12 + images/calico-cctools/versions.yaml | 4 + vm-image/README.md | 65 +++++ vm-image/build-image.sh | 60 ++++ vm-image/provision.sh | 75 +++++ 20 files changed, 1456 insertions(+), 4 deletions(-) create mode 100644 .semaphore/promotions/calico-cctools.yml create mode 100644 cctools/ccutil/secret.go create mode 100644 cctools/gce/ssh.go create mode 100644 cctools/gce/vm.go create mode 100644 cctools/subcmd/createvm/createvm.go create mode 100644 cctools/subcmd/deletevm/deletevm.go create mode 100644 cctools/subcmd/runonvm/runonvm.go create mode 100644 cctools/subcmd/secret/secret.go create mode 100644 cmd/cctools/main.go create mode 100644 images/calico-cctools/Dockerfile create mode 100644 images/calico-cctools/versions.yaml create mode 100644 vm-image/README.md create mode 100755 vm-image/build-image.sh create mode 100755 vm-image/provision.sh diff --git a/.gitignore b/.gitignore index bac9bc32..54f55133 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ cmd/bin images/calico-binfmt/bin images/calico-go-build/bin +images/calico-cctools/bin diff --git a/.semaphore/promotions/calico-cctools.yml b/.semaphore/promotions/calico-cctools.yml new file mode 100644 index 00000000..5ffecda4 --- /dev/null +++ b/.semaphore/promotions/calico-cctools.yml @@ -0,0 +1,31 @@ +version: v1.0 +name: Publish calico/cctools images +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 + +execution_time_limit: + minutes: 30 + +global_job_config: + env_vars: + - name: DEV_REGISTRIES + value: calico + secrets: + - name: docker + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - checkout + +blocks: + - name: Publish calico/cctools amd64 images + dependencies: [] + run: + when: "branch = 'master'" + task: + jobs: + - name: Linux amd64 + commands: + - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-cctools-cd CONFIRM=true; fi diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a4a7db92..48a198f4 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -48,6 +48,11 @@ promotions: pipeline_file: promotions/calico-tinygo.yml auto_promote: when: "branch = 'master' AND change_in('/images/calico-tinygo/')" + # Publish cctools images for master when the binary source or image changed. + - name: Publish calico/cctools images + pipeline_file: promotions/calico-cctools.yml + auto_promote: + when: "branch = 'master' AND change_in(['/images/calico-cctools/', '/cmd/cctools/', '/cctools/'])" blocks: - name: calico/go-build image @@ -186,3 +191,11 @@ blocks: matrix: - env_var: ARCH values: ["amd64", "arm64"] + + - name: calico/cctools image + dependencies: [] + task: + jobs: + - name: Build calico/cctools amd64 image + commands: + - make -C images calico-cctools-image ARCH=amd64 diff --git a/cctools/ccutil/secret.go b/cctools/ccutil/secret.go new file mode 100644 index 00000000..7676de19 --- /dev/null +++ b/cctools/ccutil/secret.go @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Package ccutil is the Go reimplementation of the cc-utils argoci common-scripts +// (tigera/cc-utils/argoci-images/common-scripts) that a distroless/scratch image +// can't run because it has no bash/git/ssh binaries. LocalSecret is createLocalSecret; +// the SSH/clone equivalents live alongside so the VM-lifecycle binaries are +// self-sufficient. Slated to move to a shared cc-util folder. +package ccutil + +import ( + "fmt" + "os" + "path/filepath" +) + +// LocalSecret writes the value of environment variable name to destPath (mode +// 0600, creating parent dirs) — the Go form of cc-utils' createLocalSecret, which +// materializes a mounted-secret env var to a file. Missing env var is a no-op (as +// the script is), returning found=false so callers can decide whether that's fatal. +func LocalSecret(name, destPath string) (found bool, err error) { + v, ok := os.LookupEnv(name) + if !ok { + return false, nil + } + if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil { + return false, fmt.Errorf("mkdir for %s: %w", destPath, err) + } + if err := os.WriteFile(destPath, []byte(v), 0o600); err != nil { + return false, fmt.Errorf("write secret %s: %w", destPath, err) + } + return true, nil +} + +// MustLocalSecret is LocalSecret but errors when the env var is absent — for a +// secret the caller can't proceed without (e.g. the compute SA key). +func MustLocalSecret(name, destPath string) error { + found, err := LocalSecret(name, destPath) + if err != nil { + return err + } + if !found { + return fmt.Errorf("required secret env var %q not set", name) + } + return nil +} + +// SetupComputeADC points Application Default Credentials at the compute +// service-account key so the GCP API clients authenticate. It prefers the file at +// COMPUTE_SA_KEY when that exists (a mounted secret volume); otherwise it +// materializes the key from the env var named by COMPUTE_SA_ENV (default the +// banzai SA key, an envFrom'd mounted secret) to a temp file. Either way it sets +// GOOGLE_APPLICATION_CREDENTIALS. This keeps the binary self-sufficient on a +// scratch image — no createLocalSecret step needed. +func SetupComputeADC() error { + if p := os.Getenv("COMPUTE_SA_KEY"); p != "" { + if _, err := os.Stat(p); err == nil { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", p) + return nil + } + } + name := os.Getenv("COMPUTE_SA_ENV") + if name == "" { + name = "banzai-google-service-account.json" + } + dest := filepath.Join(os.TempDir(), "compute-sa.json") + if err := MustLocalSecret(name, dest); err != nil { + return fmt.Errorf("compute SA: %w (set COMPUTE_SA_KEY to a mounted key file, or COMPUTE_SA_ENV to the key's env var name)", err) + } + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", dest) + return nil +} diff --git a/cctools/gce/ssh.go b/cctools/gce/ssh.go new file mode 100644 index 00000000..dd6e84c1 --- /dev/null +++ b/cctools/gce/ssh.go @@ -0,0 +1,332 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// SSH access to a GCE VM without gcloud: an ephemeral keypair is injected as the +// instance's `ssh-keys` metadata (the guest agent writes it into the login user's +// authorized_keys), the external IP comes from the instance itself, and the +// connection is a plain golang.org/x/crypto/ssh dial. This is what lets the run +// step drive the VM from the same scratch image createvm/deletevm run on. +package gce + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/ed25519" + "crypto/rand" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/crypto/ssh" + compute "google.golang.org/api/compute/v1" +) + +// SSH is a live connection to a VM. Close it when done. +type SSH struct { + client *ssh.Client +} + +// DialSSH injects an ephemeral keypair into the instance's metadata, reads its +// external IP, and dials SSH as user, retrying until the VM is reachable (a fresh +// VM accepts SSH only once sshd, the guest agent, and the key have caught up, so +// the first attempts routinely lose that race). This retry IS the readiness check +// createvm deliberately skips. The host key is not verified: the VM was just +// created by us, is reached only over its ephemeral external IP, and lives for +// minutes -- there is no prior key to pin. +func (c *Client) DialSSH(ctx context.Context, zone, name, user string) (*SSH, error) { + signer, authorized, err := ephemeralKey() + if err != nil { + return nil, err + } + ip, err := c.injectKeyAndGetIP(ctx, zone, name, user, authorized) + if err != nil { + return nil, err + } + + cfg := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // ephemeral CI VM, no key to pin + Timeout: 10 * time.Second, + } + addr := net.JoinHostPort(ip, "22") + + deadline := time.Now().Add(3 * time.Minute) + var lastErr error + for time.Now().Before(deadline) { + client, err := ssh.Dial("tcp", addr, cfg) + if err == nil { + return &SSH{client: client}, nil + } + lastErr = err + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(5 * time.Second): + } + } + return nil, fmt.Errorf("ssh to %s (%s) not ready after 3m: %w", name, addr, lastErr) +} + +// injectKeyAndGetIP sets the instance's ssh-keys metadata to authorize user with +// the given key (preserving any other metadata) and returns its external IP. +func (c *Client) injectKeyAndGetIP(ctx context.Context, zone, name, user, authorized string) (string, error) { + inst, err := c.svc.Instances.Get(c.project, zone, name).Context(ctx).Do() + if err != nil { + return "", fmt.Errorf("get instance %s: %w", name, err) + } + + md := inst.Metadata + if md == nil { + md = &compute.Metadata{} + } + sshKeys := fmt.Sprintf("%s:%s", user, strings.TrimSpace(authorized)) + replaced := false + for _, it := range md.Items { + if it.Key == "ssh-keys" { + it.Value = strPtr(sshKeys) + replaced = true + break + } + } + if !replaced { + md.Items = append(md.Items, &compute.MetadataItems{Key: "ssh-keys", Value: strPtr(sshKeys)}) + } + op, err := c.svc.Instances.SetMetadata(c.project, zone, name, md).Context(ctx).Do() + if err != nil { + return "", fmt.Errorf("set ssh-keys metadata on %s: %w", name, err) + } + if err := c.waitZoneOp(ctx, zone, op.Name); err != nil { + return "", fmt.Errorf("set-metadata op on %s: %w", name, err) + } + + ip := externalIP(inst) + if ip == "" { + return "", fmt.Errorf("instance %s has no external IP", name) + } + return ip, nil +} + +func externalIP(inst *compute.Instance) string { + for _, ni := range inst.NetworkInterfaces { + for _, ac := range ni.AccessConfigs { + if ac.NatIP != "" { + return ac.NatIP + } + } + } + return "" +} + +// ephemeralKey returns an ssh signer and its authorized_keys line. +func ephemeralKey() (ssh.Signer, string, error) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, "", fmt.Errorf("generate key: %w", err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + return nil, "", fmt.Errorf("signer: %w", err) + } + sshPub, err := ssh.NewPublicKey(pub) + if err != nil { + return nil, "", fmt.Errorf("public key: %w", err) + } + return signer, string(ssh.MarshalAuthorizedKey(sshPub)), nil +} + +// Close closes the underlying connection. +func (s *SSH) Close() error { return s.client.Close() } + +// Run executes cmd on the VM, streaming its stdout/stderr to the given writers, +// and returns the command's exit status. A non-zero exit is reported as exitCode +// with a nil error so the caller can propagate it; err is non-nil only for a +// connection/protocol failure. +func (s *SSH) Run(cmd string, stdout, stderr io.Writer) (exitCode int, err error) { + sess, err := s.client.NewSession() + if err != nil { + return -1, err + } + defer sess.Close() + sess.Stdout = stdout + sess.Stderr = stderr + if err := sess.Run(cmd); err != nil { + var ee *ssh.ExitError + if errors.As(err, &ee) { + return ee.ExitStatus(), nil + } + return -1, err + } + return 0, nil +} + +// PutData writes data to remote at the given mode, creating parent dirs. +func (s *SSH) PutData(data []byte, remote string, mode os.FileMode) error { + sess, err := s.client.NewSession() + if err != nil { + return err + } + defer sess.Close() + sess.Stdin = bytes.NewReader(data) + // `cat >` reads stdin; the quoting is on paths we control (no user input). + cmd := fmt.Sprintf("mkdir -p %q && cat > %q && chmod %o %q", + filepath.Dir(remote), remote, mode.Perm(), remote) + var errBuf bytes.Buffer + if err := sess.Run(cmd); err != nil { + return fmt.Errorf("put %s: %v: %s", remote, err, errBuf.String()) + } + return nil +} + +// PutFile uploads a single local file to remote at the given mode. +func (s *SSH) PutFile(local, remote string, mode os.FileMode) error { + data, err := os.ReadFile(local) + if err != nil { + return err + } + return s.PutData(data, remote, mode) +} + +// PutDir uploads a local directory tree to a remote directory (created if absent) +// by streaming a tar over the connection and untarring on the VM. +func (s *SSH) PutDir(localDir, remoteDir string) error { + sess, err := s.client.NewSession() + if err != nil { + return err + } + defer sess.Close() + + pr, pw := io.Pipe() + sess.Stdin = pr + go func() { pw.CloseWithError(tarDir(localDir, pw)) }() + + cmd := fmt.Sprintf("mkdir -p %q && tar xzf - -C %q", remoteDir, remoteDir) + if err := sess.Run(cmd); err != nil { + return fmt.Errorf("put dir %s: %w", remoteDir, err) + } + return nil +} + +// GetDir pulls a remote directory's contents into localDir (best-effort: a +// missing remote path is not an error -- an epilogue runs exactly when the files +// it wanted may never have been produced). It streams a tar off the VM. +func (s *SSH) GetDir(remoteDir, localDir string) error { + sess, err := s.client.NewSession() + if err != nil { + return err + } + defer sess.Close() + + stdout, err := sess.StdoutPipe() + if err != nil { + return err + } + // Tar the contents (not the dir itself) so they land directly under localDir. + cmd := fmt.Sprintf("cd %q 2>/dev/null && tar czf - . || true", remoteDir) + if err := sess.Start(cmd); err != nil { + return err + } + if err := untar(stdout, localDir); err != nil { + _ = sess.Wait() + return err + } + return sess.Wait() +} + +// tarDir writes a gzipped tar of dir's contents (paths relative to dir) to w. +func tarDir(dir string, w io.Writer) error { + gz := gzip.NewWriter(w) + tw := tar.NewWriter(gz) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(rel) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if info.IsDir() { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(tw, f) + return err + }) + if err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + return gz.Close() +} + +// untar extracts a gzipped tar stream into destDir, guarding against paths that +// would escape it. +func untar(r io.Reader, destDir string) error { + if err := os.MkdirAll(destDir, 0o755); err != nil { + return err + } + gz, err := gzip.NewReader(r) + if err != nil { + // An empty stream (missing remote dir) is not an error. + if errors.Is(err, io.EOF) { + return nil + } + return err + } + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + target := filepath.Join(destDir, filepath.Clean("/"+hdr.Name)) + if !strings.HasPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator)) && target != destDir { + return fmt.Errorf("tar entry escapes dest: %q", hdr.Name) + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { //nolint:gosec // CI artifacts we produced + f.Close() + return err + } + f.Close() + } + } +} diff --git a/cctools/gce/vm.go b/cctools/gce/vm.go new file mode 100644 index 00000000..04ea525f --- /dev/null +++ b/cctools/gce/vm.go @@ -0,0 +1,167 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Package gce creates, waits for, and deletes a GCE VM via the compute API — no +// gcloud CLI, so the caller can run from a scratch/distroless image. Readiness is +// reported by the VM's startup script through a guest attribute (see WaitReady), +// so create needs no SSH. +package gce + +import ( + "context" + "fmt" + "strings" + "time" + + compute "google.golang.org/api/compute/v1" +) + +// Config describes the VM to create. Zones are tried in order (capacity), and the +// first that succeeds is returned. +type Config struct { + Project string + Name string + Zones []string + MachineType string // e.g. "n2-standard-16" + DiskType string // e.g. "pd-ssd" + DiskSizeGB int64 + ImageFamily string // e.g. "ubuntu-2404-lts-amd64" + ImageProject string // e.g. "ubuntu-os-cloud" + MaxRun time.Duration + Labels map[string]string + StartupScript string // bash run as the GCE startup-script (on the VM) +} + +// Client is a compute API client scoped to one project. +type Client struct { + svc *compute.Service + project string +} + +// New builds a client using Application Default Credentials. Point +// GOOGLE_APPLICATION_CREDENTIALS at the service-account key (a mounted secret) +// before calling -- the cmd wrappers do this from COMPUTE_SA_KEY. +func New(ctx context.Context, project string) (*Client, error) { + svc, err := compute.NewService(ctx) + if err != nil { + return nil, fmt.Errorf("compute client: %w", err) + } + return &Client{svc: svc, project: project}, nil +} + +// Create inserts the instance in the first zone that accepts it and waits for the +// insert operation to finish, returning that zone. The VM gets an external IP, +// cloud-platform scope, guest attributes enabled, and a max-run-duration whose +// deadline GCP reclaims the VM at (DELETE) — a leaked-VM backstop independent of +// any cleanup step. +func (c *Client) Create(ctx context.Context, cfg Config) (zone string, err error) { + var lastErr error + for _, z := range cfg.Zones { + inst := c.instanceSpec(z, cfg) + op, err := c.svc.Instances.Insert(cfg.Project, z, inst).Context(ctx).Do() + if err != nil { + lastErr = fmt.Errorf("insert in %s: %w", z, err) + continue + } + if err := c.waitZoneOp(ctx, z, op.Name); err != nil { + lastErr = fmt.Errorf("insert op in %s: %w", z, err) + continue + } + return z, nil + } + if lastErr == nil { + lastErr = fmt.Errorf("no zones configured") + } + return "", fmt.Errorf("could not create %s in any zone: %w", cfg.Name, lastErr) +} + +func (c *Client) instanceSpec(zone string, cfg Config) *compute.Instance { + inst := &compute.Instance{ + Name: cfg.Name, + MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, cfg.MachineType), + Labels: cfg.Labels, + Disks: []*compute.AttachedDisk{{ + Boot: true, + AutoDelete: true, + InitializeParams: &compute.AttachedDiskInitializeParams{ + SourceImage: fmt.Sprintf("projects/%s/global/images/family/%s", cfg.ImageProject, cfg.ImageFamily), + DiskSizeGb: cfg.DiskSizeGB, + DiskType: fmt.Sprintf("zones/%s/diskTypes/%s", zone, cfg.DiskType), + }, + }}, + // One NAT access config = an ephemeral external IP (SSH from the run step). + NetworkInterfaces: []*compute.NetworkInterface{{ + AccessConfigs: []*compute.AccessConfig{{Type: "ONE_TO_ONE_NAT", Name: "External NAT"}}, + }}, + ServiceAccounts: []*compute.ServiceAccount{{ + Email: "default", + Scopes: []string{compute.CloudPlatformScope}, + }}, + } + // A startup script is optional: the ci-base image already has docker etc., so + // createvm passes none. Only set it for a stock image that needs provisioning. + if cfg.StartupScript != "" { + inst.Metadata = &compute.Metadata{Items: []*compute.MetadataItems{ + {Key: "startup-script", Value: strPtr(cfg.StartupScript)}, + }} + } + if cfg.MaxRun > 0 { + inst.Scheduling = &compute.Scheduling{ + ProvisioningModel: "STANDARD", + InstanceTerminationAction: "DELETE", + MaxRunDuration: &compute.Duration{Seconds: int64(cfg.MaxRun.Seconds())}, + } + } + return inst +} + +// Delete removes the instance and waits for the delete operation to finish. A +// not-found instance is treated as already deleted. +func (c *Client) Delete(ctx context.Context, zone, name string) error { + op, err := c.svc.Instances.Delete(c.project, zone, name).Context(ctx).Do() + if err != nil { + if isNotFound(err) { + return nil + } + return fmt.Errorf("delete %s in %s: %w", name, zone, err) + } + return c.waitZoneOp(ctx, zone, op.Name) +} + +// FindZone returns the zone an instance of this name lives in, or "" if none. +// Lets a zone-agnostic cleanup delete a VM without carrying its zone. +func (c *Client) FindZone(ctx context.Context, name string) (string, error) { + agg, err := c.svc.Instances.AggregatedList(c.project).Filter("name=" + name).Context(ctx).Do() + if err != nil { + return "", err + } + for scope, list := range agg.Items { + if len(list.Instances) == 0 { + continue + } + // scope is "zones/". + return strings.TrimPrefix(scope, "zones/"), nil + } + return "", nil +} + +// waitZoneOp blocks until a zone operation reaches DONE, surfacing its error. +func (c *Client) waitZoneOp(ctx context.Context, zone, op string) error { + for { + got, err := c.svc.ZoneOperations.Wait(c.project, zone, op).Context(ctx).Do() + if err != nil { + return err + } + if got.Status == "DONE" { + if got.Error != nil && len(got.Error.Errors) > 0 { + return fmt.Errorf("%s: %s", got.Error.Errors[0].Code, got.Error.Errors[0].Message) + } + return nil + } + } +} + +func isNotFound(err error) bool { + return err != nil && strings.Contains(err.Error(), "notFound") +} + +func strPtr(s string) *string { return &s } diff --git a/cctools/subcmd/createvm/createvm.go b/cctools/subcmd/createvm/createvm.go new file mode 100644 index 00000000..219b318b --- /dev/null +++ b/cctools/subcmd/createvm/createvm.go @@ -0,0 +1,115 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Package createvm creates the CI GCE VM (over the compute API) from the ci-base +// image — docker/go/kind/kubectl/gh prebaked, so no startup script — and writes +// its zone to ZONE_OUT for the next workflow step. It does NOT wait or SSH: the +// run step's SSH connecting is the readiness check. Runs from the cctools scratch +// image (no gcloud, no bash). Config comes from env vars the workflow sets; the +// compute SA is a mounted key file (COMPUTE_SA_KEY) or its env var (see ccutil). +package createvm + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/projectcalico/go-build/cctools/ccutil" + "github.com/projectcalico/go-build/cctools/gce" +) + +// Run executes the createvm subcommand and returns its exit code. +func Run() int { + if err := run(context.Background()); err != nil { + fmt.Fprintf(os.Stderr, "createvm: %v\n", err) + return 1 + } + return 0 +} + +func run(ctx context.Context) error { + name := mustEnv("VM_NAME") + project := envOr("GCP_VM_PROJECT", "unique-caldron-775") + zoneOut := envOr("ZONE_OUT", "/tmp/vm-zone") + // Point ADC at the compute SA (mounted key file, or materialized from its env var). + if err := ccutil.SetupComputeADC(); err != nil { + return err + } + + maxRun, err := time.ParseDuration(envOr("GOOGLE_VM_MAX_RUN_DURATION", "90m")) + if err != nil { + return fmt.Errorf("GOOGLE_VM_MAX_RUN_DURATION: %w", err) + } + diskGB, err := parseDiskGB(envOr("GOOGLE_VM_DISK_SIZE", "200GB")) + if err != nil { + return err + } + + client, err := gce.New(ctx, project) + if err != nil { + return err + } + + cfg := gce.Config{ + Project: project, + Name: name, + Zones: strings.Fields(envOr("GOOGLE_VM_ZONES", "us-central1-a us-central1-b us-central1-c us-central1-f")), + MachineType: envOr("GOOGLE_VM_MACHINE_TYPE", "n2-standard-16"), + DiskType: envOr("GOOGLE_VM_DISK_TYPE", "pd-ssd"), + DiskSizeGB: diskGB, + // Default to the ci-base custom image (docker/go/kind/kubectl/gh baked in, + // built by cloud/kind-rig/vm-image/build-image.sh), so the VM boots ready and + // run-kindrig.sh does no tool installs. Override for a stock-ubuntu VM. + ImageFamily: envOr("GOOGLE_VM_IMAGE_FAMILY", "ci-base"), + ImageProject: envOr("GOOGLE_VM_IMAGE_PROJECT", "unique-caldron-775"), + MaxRun: maxRun, + Labels: map[string]string{ + "ci-runner": "true", + "ci-project": "kindrig", + "ci-workflow": envOr("CI_WORKFLOW_LABEL", "unknown"), + }, + } + + fmt.Printf("[createvm] creating %s (%s) in %s across %v\n", name, cfg.MachineType, project, cfg.Zones) + zone, err := client.Create(ctx, cfg) + if err != nil { + return err + } + if err := os.WriteFile(zoneOut, []byte(zone), 0o644); err != nil { + return fmt.Errorf("write zone to %s: %w", zoneOut, err) + } + // Create-only: readiness is the run-e2e step's job -- it retries SSH until the + // VM is reachable (which it must, to ship + run), so that connect IS the + // readiness check. The startup script (docker install) runs meanwhile. + fmt.Printf("[createvm] %s created in %s (zone -> %s)\n", name, zone, zoneOut) + return nil +} + +// parseDiskGB accepts "200GB", "200G", or "200" and returns the GB count. +func parseDiskGB(s string) (int64, error) { + s = strings.TrimSpace(s) + s = strings.TrimSuffix(strings.TrimSuffix(strings.ToUpper(s), "GB"), "G") + n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + if err != nil { + return 0, fmt.Errorf("GOOGLE_VM_DISK_SIZE %q: want e.g. 200GB: %w", s, err) + } + return n, nil +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func mustEnv(key string) string { + v := os.Getenv(key) + if v == "" { + fmt.Fprintf(os.Stderr, "createvm: %s must be set\n", key) + os.Exit(1) + } + return v +} diff --git a/cctools/subcmd/deletevm/deletevm.go b/cctools/subcmd/deletevm/deletevm.go new file mode 100644 index 00000000..37a9adbc --- /dev/null +++ b/cctools/subcmd/deletevm/deletevm.go @@ -0,0 +1,73 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Package deletevm deletes the kind-rig CI GCE VM by name (finding its zone if +// ZONE isn't given), over the compute API — no gcloud. Best-effort: the VM's +// max-run-duration is the ultimate backstop, so a failure here is logged, not +// fatal. Runs from the same scratch image as createvm, in the workflow's onExit +// cleanup step. +package deletevm + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/projectcalico/go-build/cctools/ccutil" + "github.com/projectcalico/go-build/cctools/gce" +) + +// Run executes the deletevm subcommand and returns its exit code. Best-effort: +// any failure returns 0 (the VM's max-run-duration is the backstop). +func Run() int { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + name := os.Getenv("VM_NAME") + if name == "" { + fmt.Fprintln(os.Stderr, "deletevm: VM_NAME must be set") + return 1 + } + project := envOr("GCP_VM_PROJECT", "unique-caldron-775") + // Materialize the compute SA from its env var (COMPUTE_SA_ENV) -> ADC, exactly + // like createvm. The cleanup step injects the SA as an env var (banzai-secrets), + // NOT a mounted file -- pointing ADC at a nonexistent /secrets file made every + // delete fail auth, leaking VMs to the max-run-duration backstop. + if err := ccutil.SetupComputeADC(); err != nil { + fmt.Fprintf(os.Stderr, "deletevm: %v (leaving to max-run-duration)\n", err) + return 0 + } + + client, err := gce.New(ctx, project) + if err != nil { + fmt.Fprintf(os.Stderr, "deletevm: %v (leaving to max-run-duration)\n", err) + return 0 + } + + zone := os.Getenv("ZONE") + if zone == "" { + if zone, err = client.FindZone(ctx, name); err != nil { + fmt.Fprintf(os.Stderr, "deletevm: find zone for %s: %v (leaving to max-run-duration)\n", name, err) + return 0 + } + } + if zone == "" { + fmt.Printf("[deletevm] no VM %s found; nothing to delete\n", name) + return 0 + } + + fmt.Printf("[deletevm] deleting %s in %s\n", name, zone) + if err := client.Delete(ctx, zone, name); err != nil { + fmt.Fprintf(os.Stderr, "deletevm: %v (leaving to max-run-duration)\n", err) + return 0 + } + fmt.Printf("[deletevm] deleted %s\n", name) + return 0 +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/cctools/subcmd/runonvm/runonvm.go b/cctools/subcmd/runonvm/runonvm.go new file mode 100644 index 00000000..ea42e154 --- /dev/null +++ b/cctools/subcmd/runonvm/runonvm.go @@ -0,0 +1,234 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Package runonvm runs a script on a GCE VM over SSH, with no gcloud -- the +// generic "runOn: vm" primitive for the cctools scratch image. It ships files and +// secrets to the VM, runs a script there (a FILE, never a command string, so +// nothing has to survive three levels of shell quoting), pulls artifacts back on +// ANY exit, and exits with the script's own status. +// +// It is meant to be the `command` of an Argo `script` template: Argo writes the +// template's `source:` to a temp file and appends its path as the final argument, +// so `runonvm ` ships that source to the VM and runs it there +// -- i.e. the workflow's `source:` block executes naturally on the VM. (--script +// is the equivalent for direct/CLI use.) +// +// The VM (its name/zone/project) comes from env, same as createvm/deletevm; the +// compute SA is materialized by ccutil. Everything job-specific -- which files, +// which secrets, which artifacts -- is a flag, so this stays generic across CI +// jobs. +// +// command: [runonvm, +// --put-env, ENV_VAR:remote/path, # repeatable; env value -> 0600 file +// --env, ENV_VAR, # repeatable; forwarded into a sourced env file +// --get, remote/dir:local/dir] # repeatable; best-effort, on exit +// source: | # Argo appends this file; it runs on the VM +// ... +package runonvm + +import ( + "context" + "flag" + "fmt" + "os" + "path" + "strings" + + "github.com/projectcalico/go-build/cctools/ccutil" + "github.com/projectcalico/go-build/cctools/gce" +) + +type stringList []string + +func (s *stringList) String() string { return strings.Join(*s, ",") } +func (s *stringList) Set(v string) error { + *s = append(*s, v) + return nil +} + +// Run executes the runonvm subcommand and returns its exit code. +func Run() int { + return run(context.Background()) +} + +func run(ctx context.Context) int { + var puts, putEnvs, envs, gets stringList + scriptFlag := flag.String("script", "", "script to run on the VM (Argo appends the source file as a trailing arg; --script is the CLI equivalent)") + user := flag.String("user", envOr("VM_SSH_USER", "ubuntu"), "SSH user") + flag.Var(&puts, "put", "LOCAL:REMOTE file or dir to upload before running (repeatable)") + flag.Var(&putEnvs, "put-env", "ENVVAR:REMOTE -- write an env var's value to a 0600 remote file (repeatable)") + flag.Var(&envs, "env", "ENVVAR to forward into an env file the script sources before running (repeatable)") + flag.Var(&gets, "get", "REMOTE:LOCAL dir to pull back on exit, best-effort (repeatable)") + flag.Parse() + + // Argo passes the script template's staged source file as the trailing arg; + // --script is the equivalent for direct use. + script := *scriptFlag + if flag.NArg() > 0 { + script = flag.Arg(0) + } + if script == "" { + fmt.Fprintln(os.Stderr, "runonvm: need a script (trailing arg or --script)") + return 2 + } + + name := mustEnv("VM_NAME") + project := envOr("GCP_VM_PROJECT", "unique-caldron-775") + + if err := ccutil.SetupComputeADC(); err != nil { + fmt.Fprintf(os.Stderr, "runonvm: %v\n", err) + return 1 + } + client, err := gce.New(ctx, project) + if err != nil { + fmt.Fprintf(os.Stderr, "runonvm: %v\n", err) + return 1 + } + + zone := os.Getenv("ZONE") + if zone == "" { + zone, err = client.FindZone(ctx, name) + if err != nil || zone == "" { + fmt.Fprintf(os.Stderr, "runonvm: could not find zone for %s (set ZONE): %v\n", name, err) + return 1 + } + } + + fmt.Printf("[runonvm] connecting to %s in %s\n", name, zone) + conn, err := client.DialSSH(ctx, zone, name, *user) + if err != nil { + fmt.Fprintf(os.Stderr, "runonvm: %v\n", err) + return 1 + } + defer conn.Close() + + // Pull artifacts back on ANY exit, so a mid-run failure still returns logs. + defer func() { + for _, g := range gets { + remote, local, ok := splitPair(g) + if !ok { + continue + } + if err := conn.GetDir(remote, local); err != nil { + fmt.Fprintf(os.Stderr, "[runonvm] get %s: %v (continuing)\n", remote, err) + } else { + fmt.Printf("[runonvm] pulled %s -> %s\n", remote, local) + } + } + }() + + // Ship files and secrets before running. + for _, p := range puts { + local, remote, ok := splitPair(p) + if !ok { + fmt.Fprintf(os.Stderr, "runonvm: bad --put %q (want LOCAL:REMOTE)\n", p) + return 2 + } + info, err := os.Stat(local) + if err != nil { + fmt.Fprintf(os.Stderr, "runonvm: --put %s: %v\n", local, err) + return 1 + } + if info.IsDir() { + err = conn.PutDir(local, remote) + } else { + err = conn.PutFile(local, remote, 0o644) + } + if err != nil { + fmt.Fprintf(os.Stderr, "runonvm: %v\n", err) + return 1 + } + fmt.Printf("[runonvm] put %s -> %s\n", local, remote) + } + for _, pe := range putEnvs { + envVar, remote, ok := splitPair(pe) + if !ok { + fmt.Fprintf(os.Stderr, "runonvm: bad --put-env %q (want ENVVAR:REMOTE)\n", pe) + return 2 + } + val, present := os.LookupEnv(envVar) + if !present { + fmt.Fprintf(os.Stderr, "runonvm: --put-env %s: env var not set\n", envVar) + return 1 + } + if err := conn.PutData([]byte(val), remote, 0o600); err != nil { + fmt.Fprintf(os.Stderr, "runonvm: %v\n", err) + return 1 + } + fmt.Printf("[runonvm] put-env %s -> %s (0600)\n", envVar, remote) + } + + // Forward selected env vars into a file the script sources before running -- + // the generic path for a commit SHA, tokens, etc. (mirrors argoci's /tmp/secrets). + // Values are shell-quoted, so any content is safe. An unset var is skipped + // (lenient: the script decides whether a missing one is fatal). + remoteEnv := "/tmp/runonvm.env" + haveEnv := false + if len(envs) > 0 { + var b strings.Builder + for _, name := range envs { + val, present := os.LookupEnv(name) + if !present { + fmt.Fprintf(os.Stderr, "[runonvm] --env %s not set, skipping\n", name) + continue + } + fmt.Fprintf(&b, "export %s=%s\n", name, shellQuote(val)) + haveEnv = true + } + if haveEnv { + if err := conn.PutData([]byte(b.String()), remoteEnv, 0o600); err != nil { + fmt.Fprintf(os.Stderr, "runonvm: write env file: %v\n", err) + return 1 + } + } + } + + // Upload the script to a temp path and run it (a file, not a command string). + remoteScript := path.Join("/tmp", path.Base(script)) + if err := conn.PutFile(script, remoteScript, 0o755); err != nil { + fmt.Fprintf(os.Stderr, "runonvm: upload script: %v\n", err) + return 1 + } + runCmd := "bash " + remoteScript + if haveEnv { + runCmd = fmt.Sprintf(". %s && bash %s", remoteEnv, remoteScript) + } + fmt.Printf("[runonvm] running %s on %s\n", remoteScript, name) + code, err := conn.Run(runCmd, os.Stdout, os.Stderr) + if err != nil { + fmt.Fprintf(os.Stderr, "runonvm: run: %v\n", err) + return 1 + } + fmt.Printf("[runonvm] script finished (rc=%d)\n", code) + return code +} + +// shellQuote single-quotes s for safe use in a POSIX shell, escaping embedded +// single quotes as '\”. +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// splitPair splits "A:B" on the first colon. +func splitPair(s string) (a, b string, ok bool) { + i := strings.IndexByte(s, ':') + if i < 0 { + return "", "", false + } + return s[:i], s[i+1:], true +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func mustEnv(key string) string { + v := os.Getenv(key) + if v == "" { + fmt.Fprintf(os.Stderr, "runonvm: %s must be set\n", key) + os.Exit(2) + } + return v +} diff --git a/cctools/subcmd/secret/secret.go b/cctools/subcmd/secret/secret.go new file mode 100644 index 00000000..c6121ea1 --- /dev/null +++ b/cctools/subcmd/secret/secret.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Package secret materializes a mounted-secret env var to a file — the Go form of +// cc-utils' createLocalSecret, for the scratch image. Usage: secret NAME PATH. +package secret + +import ( + "fmt" + "os" + + "github.com/projectcalico/go-build/cctools/ccutil" +) + +// Run executes the secret subcommand and returns its exit code. +func Run() int { + if len(os.Args) != 3 { + fmt.Fprintln(os.Stderr, "usage: secret ") + return 2 + } + found, err := ccutil.LocalSecret(os.Args[1], os.Args[2]) + if err != nil { + fmt.Fprintf(os.Stderr, "secret: %v\n", err) + return 1 + } + if !found { + fmt.Printf("secret %s not created (env var not set)\n", os.Args[1]) + return 0 + } + fmt.Printf("created %s\n", os.Args[2]) + return 0 +} diff --git a/cmd/Makefile b/cmd/Makefile index 5dc102a4..9f05ff5d 100644 --- a/cmd/Makefile +++ b/cmd/Makefile @@ -7,7 +7,7 @@ BINFMT_VERSION = $(shell yq -r '.tonistiigi-binfmt.version' ../images/calico-bin QEMU_VERSION = $(shell yq -r '.qemu.version' ../images/calico-binfmt/versions.yaml) .PHONY: build -build: bin/binfmt-$(ARCH) bin/semvalidator-$(ARCH) +build: bin/binfmt-$(ARCH) bin/semvalidator-$(ARCH) bin/cctools-$(ARCH) bin/binfmt-$(ARCH): binfmt/main.go CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) \ @@ -17,6 +17,11 @@ bin/semvalidator-$(ARCH): semvalidator/main.go CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) \ go build -o bin/semvalidator-$(ARCH) -v -buildvcs=false -ldflags "-s -w" semvalidator/main.go +# cctools: the single CI VM binary (createvm/deletevm/secret/runonvm subcommands). +bin/cctools-$(ARCH): cctools/main.go + CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) \ + go build -o bin/cctools-$(ARCH) -v -buildvcs=false -ldflags "-s -w" ./cctools + .PHONY: clean clean: rm -fr bin/ diff --git a/cmd/cctools/main.go b/cmd/cctools/main.go new file mode 100644 index 00000000..f8c0430d --- /dev/null +++ b/cmd/cctools/main.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Command cctools bundles the CI VM helpers into a single binary, dispatched by +// subcommand -- one image, different args, not four binaries: +// +// cctools createvm create the CI GCE VM (config from env) +// cctools deletevm delete it by name (best-effort cleanup) +// cctools secret materialize a mounted-secret env var to a file +// cctools runonvm [flags]