diff --git a/cmd/gmp-migrate/Dockerfile b/cmd/gmp-migrate/Dockerfile new file mode 100644 index 0000000000..9132d80b1f --- /dev/null +++ b/cmd/gmp-migrate/Dockerfile @@ -0,0 +1,55 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM google-go.pkg.dev/golang:1.26.4@sha256:3444149d0a7e3f7cfb9c2db65f0f75676fe6ad04de3ce72674efb120c08dd1c1 AS buildbase +ARG TARGETOS +ARG TARGETARCH +ARG BUILDARCH +WORKDIR /app +COPY charts/values.global.yaml charts/values.global.yaml +COPY go.mod go.mod +COPY go.sum go.sum +COPY tools tools +# Copy the Go vendor directory only if it exists. Vendor folder will automatically +# cause 'go build' to use -mod=vendor flag (otherwise -mod=mod is used). +COPY vendor* vendor +COPY cmd cmd +COPY pkg pkg + +ENV GOEXPERIMENT=boringcrypto +ENV CGO_ENABLED=1 +ENV GOFIPS140=off +ENV GOTOOLCHAIN=local +ENV GOOS=${TARGETOS} +ENV GOARCH=${TARGETARCH} +RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \ + apt-get update && apt-get install -y --no-install-recommends \ + gcc-aarch64-linux-gnu libc6-dev-arm64-cross; \ + export CC=aarch64-linux-gnu-gcc; \ + elif [ "${TARGETARCH}" = "amd64" ] && [ "${BUILDARCH}" != "amd64" ]; then \ + apt-get update && apt-get install -y --no-install-recommends \ + gcc-x86-64-linux-gnu libc6-dev-amd64-cross; \ + export CC=x86_64-linux-gnu-gcc; \ + fi && \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build \ + -ldflags="-X github.com/prometheus/common/version.Version=$(cat charts/values.global.yaml | go tool -modfile="tools/go.mod" yq '.version' ) \ + -X github.com/prometheus/common/version.BuildDate=$(date --iso-8601=seconds)" \ + -o gmp-migrate \ + cmd/gmp-migrate/*.go + + +FROM gke.gcr.io/gke-distroless/libc:gke_distroless_20260307.00_p0@sha256:d5c073079125b887158bb1dd0ee4da49b39a08203c3c96124ee310962dd5aae2 +COPY --from=buildbase /app/gmp-migrate /bin/gmp-migrate +ENTRYPOINT ["/bin/gmp-migrate"] diff --git a/cmd/gmp-migrate/README.md b/cmd/gmp-migrate/README.md new file mode 100644 index 0000000000..315c313cf1 --- /dev/null +++ b/cmd/gmp-migrate/README.md @@ -0,0 +1,6 @@ +# gmp-migrate + +`gmp-migrate` is a tool designed to migrate Prometheus Operator resources to Google Cloud Managed Service for Prometheus (GMP) resources. + +> [!WARNING] +> This tool is currently **experimental** and a **work-in-progress (WIP)**. It is not ready for use. diff --git a/cmd/gmp-migrate/main.go b/cmd/gmp-migrate/main.go new file mode 100644 index 0000000000..4ae8126afd --- /dev/null +++ b/cmd/gmp-migrate/main.go @@ -0,0 +1,103 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "log/slog" + "os" + "strings" + + "github.com/GoogleCloudPlatform/prometheus-engine/pkg/migrate" +) + +// commaStringSlice implements the flag.Value interface to support +// repeated and/or comma-separated string flags. +type commaStringSlice []string + +func (s *commaStringSlice) String() string { + return strings.Join(*s, ",") +} + +func (s *commaStringSlice) Set(value string) error { + for p := range strings.SplitSeq(value, ",") { + trimmed := strings.TrimSpace(p) + if trimmed != "" { + *s = append(*s, trimmed) + } + } + return nil +} + +func main() { + slog.SetDefault(slog.New(migrate.NewConsoleHandler(os.Stderr))) + + var inputFiles commaStringSlice + flag.Var(&inputFiles, "file", "Input source (YAML file, directory, or '-' for stdin) (Required)") + flag.Var(&inputFiles, "f", "Input source (YAML file, directory, or '-' for stdin) (Required)") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) + fmt.Fprint(os.Stderr, "Migrate Prometheus Operator configurations to Google Managed Prometheus (GMP).\n\n") + flag.PrintDefaults() + } + flag.Parse() + + // Reject unexpected positional arguments to prevent silent typos (like forgetting -f) + if flag.NArg() > 0 { + slog.Error("Unexpected positional arguments.", + slog.Any("arguments", flag.Args()), + ) + fmt.Fprintln(os.Stderr, "\nAll input files and directories must be explicitly passed using the -f or --file flags. For example:") + var allInputs []string + allInputs = append(allInputs, inputFiles...) + allInputs = append(allInputs, flag.Args()...) + fmt.Fprintf(os.Stderr, " %s -f %s\n\n", os.Args[0], strings.Join(allInputs, " -f ")) + flag.Usage() + os.Exit(1) + } + + if len(inputFiles) == 0 { + slog.Error("Flag -f / --file is required.") + flag.Usage() + os.Exit(1) + } + + migrator := migrate.NewMigrator() + report, err := migrator.Run(inputFiles...) + if err != nil { + slog.Error("Migration failed", slog.Any("error", err)) + os.Exit(1) + } + + // If any resource failed to convert in-memory, we print summary and abort + if report.FailedCount > 0 { + migrator.PrintSummary(report) // Still print the diagnostic summary to Stderr + slog.Error("Migration aborted: resources failed conversion. Zero manifests were written to Stdout.", + slog.Int("failures", report.FailedCount), + ) + os.Exit(1) + } + + // Write the converted GMP manifests using the migrator's Stdout stream + if err := migrator.WriteOutputs(report.Outputs); err != nil { + slog.Error("Failed to write outputs", slog.Any("error", err)) + os.Exit(1) + } + + // Print the successful complete summary to Stderr + migrator.PrintSummary(report) +} diff --git a/go.mod b/go.mod index ae0cc8bbed..5c57719baf 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,10 @@ require ( sigs.k8s.io/controller-runtime v0.18.7 ) -require github.com/efficientgo/e2e v0.14.1-0.20230710114240-c316eb95ae5b +require ( + github.com/efficientgo/e2e v0.14.1-0.20230710114240-c316eb95ae5b + sigs.k8s.io/yaml v1.6.0 +) require ( cloud.google.com/go/auth v0.16.5 // indirect @@ -83,7 +86,7 @@ require ( github.com/edsrzf/mmap-go v1.2.0 // indirect github.com/efficientgo/core v1.0.0-rc.3 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/fatih/color v1.18.0 // indirect @@ -114,7 +117,7 @@ require ( github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect github.com/hashicorp/go-version v1.7.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect @@ -176,7 +179,6 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index f413a04782..0e7bd85142 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,8 @@ github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3re github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM= @@ -249,8 +249,8 @@ github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hetznercloud/hcloud-go/v2 v2.9.0 h1:s0N6R7Zoi2DPfMtUF5o9VeUBzTtHVY6MIkHOQnfu/AY= github.com/hetznercloud/hcloud-go/v2 v2.9.0/go.mod h1:qtW/TuU7Bs16ibXl/ktJarWqU2LwHr7eGlwoilHxtgg= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/ionos-cloud/sdk-go/v6 v6.1.11 h1:J/uRN4UWO3wCyGOeDdMKv8LWRzKu6UIkLEaes38Kzh8= github.com/ionos-cloud/sdk-go/v6 v6.1.11/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= diff --git a/pkg/migrate/logger.go b/pkg/migrate/logger.go new file mode 100644 index 0000000000..78afec5182 --- /dev/null +++ b/pkg/migrate/logger.go @@ -0,0 +1,210 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + "fmt" + "io" + "log/slog" + "maps" + "os" + "strings" + "sync" +) + +// ResourceStatus defines the final migration state of an ingested resource. +type ResourceStatus int + +const ( + StatusSuccess ResourceStatus = iota // 0 (Migrated Successfully) + StatusSkipped // 1 (Skipped / Unsupported) + StatusWarning // 2 (Migrated with Warnings) + StatusFailed // 3 (Failed) +) + +// statusLevels maps slog.Levels to their corresponding ResourceStatus. +// Levels omitted from this map (like slog.LevelInfo) represent progress logs +// and are ignored for status tracking. +var statusLevels = map[slog.Level]ResourceStatus{ + slog.LevelDebug: StatusSuccess, + slog.LevelWarn: StatusWarning, + slog.LevelError: StatusFailed, +} + +// loggerState encapsulates the shared, thread-safe state across all handler clones. +type loggerState struct { + mu sync.Mutex + resourceStatuses map[string]ResourceStatus +} + +// ConsoleHandler is a thread-safe slog.Handler that formats logs for the console (Stderr) +// and tracks the highest log level seen per resource (for statistics). +type ConsoleHandler struct { + out io.Writer + state *loggerState + attrs []slog.Attr +} + +// NewConsoleHandler creates a new ConsoleHandler. +func NewConsoleHandler(out io.Writer) *ConsoleHandler { + if out == nil { + out = os.Stderr + } + return &ConsoleHandler{ + out: out, + state: &loggerState{ + resourceStatuses: make(map[string]ResourceStatus), + }, + } +} + +func (h *ConsoleHandler) Enabled(_ context.Context, _ slog.Level) bool { + return true // Log everything +} + +func (h *ConsoleHandler) Handle(_ context.Context, r slog.Record) error { + h.state.mu.Lock() + defer h.state.mu.Unlock() + + var kind, namespace, name, file, migrationStatus string + var extraAttrs []string + + // Helper to process and categorize attributes + processAttr := func(a slog.Attr) { + val := a.Value.Resolve() + switch a.Key { + case "kind": + kind = val.String() + case "namespace": + namespace = val.String() + case "name": + name = val.String() + case "file": + file = val.String() + case "migration_status": + migrationStatus = val.String() + default: + // Collect all other attributes to print at the end of the line + extraAttrs = append(extraAttrs, fmt.Sprintf("%s=%v", a.Key, val.Any())) + } + } + + // Extract attributes bound to the logger instance + for _, a := range h.attrs { + processAttr(a) + } + + // Extract attributes passed in the individual log call + r.Attrs(func(a slog.Attr) bool { + processAttr(a) + return true + }) + + // Map slog.Level to string for console output. + var levelStr string + switch r.Level { + case slog.LevelDebug: + levelStr = "SUCCESS" + case slog.LevelInfo: + levelStr = "INFO" + if migrationStatus == "skipped" { + levelStr = "SKIPPED" + } + case slog.LevelWarn: + levelStr = "WARNING" + case slog.LevelError: + levelStr = "ERROR" + default: + levelStr = r.Level.String() + } + + // Format prefix cleanly + var prefix string + if file != "" { + prefix = fmt.Sprintf("[%s] ", file) + } else if kind != "" && name != "" { + if namespace == "" { + prefix = fmt.Sprintf("[%s:%s] ", kind, name) + } else { + prefix = fmt.Sprintf("[%s:%s/%s] ", kind, namespace, name) + } + } + + // 1. Write formatted log to Stderr (console), appending extra attributes if any. + var suffix string + if len(extraAttrs) > 0 { + suffix = " " + strings.Join(extraAttrs, " ") + } + consoleLine := fmt.Sprintf("[%s] %s%s%s\n", levelStr, prefix, r.Message, suffix) + if _, err := io.WriteString(h.out, consoleLine); err != nil { + return err + } + + // 2. Track the migration status of the resource (for final report) + var key string + if kind != "" && name != "" { + if namespace == "" { + key = fmt.Sprintf("%s/%s", kind, name) + } else { + key = fmt.Sprintf("%s/%s/%s", kind, namespace, name) + } + } else if file != "" { + key = file + } + + if key != "" { + if r.Level == slog.LevelInfo && migrationStatus == "skipped" { + h.trackStatus(key, StatusSkipped) + } else if status, ok := statusLevels[r.Level]; ok { + h.trackStatus(key, status) + } + } + + return nil +} + +func (h *ConsoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + if len(attrs) == 0 { + return h + } + newAttrs := make([]slog.Attr, 0, len(h.attrs)+len(attrs)) + newAttrs = append(newAttrs, h.attrs...) + newAttrs = append(newAttrs, attrs...) + return &ConsoleHandler{ + out: h.out, + state: h.state, + attrs: newAttrs, + } +} + +func (h *ConsoleHandler) WithGroup(_ string) slog.Handler { + return h +} + +// ResourceStatuses returns a thread-safe copy of the tracked resource statuses. +func (h *ConsoleHandler) ResourceStatuses() map[string]ResourceStatus { + h.state.mu.Lock() + defer h.state.mu.Unlock() + + return maps.Clone(h.state.resourceStatuses) +} + +// trackStatus updates the tracked status for a key if the new status is more severe. +func (h *ConsoleHandler) trackStatus(key string, status ResourceStatus) { + if val, exists := h.state.resourceStatuses[key]; !exists || status > val { + h.state.resourceStatuses[key] = status + } +} diff --git a/pkg/migrate/migrate.go b/pkg/migrate/migrate.go new file mode 100644 index 0000000000..1055c5c4a3 --- /dev/null +++ b/pkg/migrate/migrate.go @@ -0,0 +1,355 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + + "maps" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" +) + +// MigrationReport accumulates the statistics and payloads of the migration run. +type MigrationReport struct { + SuccessCount int // Successfully migrated with no warnings + WarningCount int // Successfully migrated but had warnings + SkippedCount int // Bypassed because resource is unsupported/out-of-scope + FailedCount int // Fatal failure, resource skipped + Outputs []*unstructured.Unstructured // Converted GMP manifests in-memory +} + +// Migrator orchestrates the migration process. +type Migrator struct { + converters map[string]ResourceConverter + cache *ResourceCache + + // Decoupled streams (defaults to os.Stdin/os.Stdout/os.Stderr) + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + logger *slog.Logger +} + +// NewMigrator creates a new Migrator. +func NewMigrator() *Migrator { + return &Migrator{ + converters: make(map[string]ResourceConverter), + cache: NewResourceCache(), + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + logger: slog.Default(), + } +} + +// RegisterConverter registers a converter for a specific resource Kind. +func (m *Migrator) RegisterConverter(c ResourceConverter) { + if m.converters == nil { + m.converters = make(map[string]ResourceConverter) + } + m.converters[c.ImportKey()] = c +} + +// Run executes the migration flow and returns the summary report across multiple inputs. +func (m *Migrator) Run(inputPaths ...string) (*MigrationReport, error) { + if m.Stdin == nil { + m.Stdin = os.Stdin + } + if m.Stdout == nil { + m.Stdout = os.Stdout + } + if m.Stderr == nil { + m.Stderr = os.Stderr + } + + if m.converters == nil { + m.converters = make(map[string]ResourceConverter) + } + + m.cache = NewResourceCache() + + report := &MigrationReport{} + + // Instantiate our custom ConsoleHandler + handler := NewConsoleHandler(m.Stderr) + m.logger = slog.New(handler) + + // 1. Parse all inputs + for _, path := range inputPaths { + if err := m.parseInputs(path); err != nil { + return nil, fmt.Errorf("failed to parse input %q: %w", path, err) + } + } + + // 2. Run converters across the cached resources + outputs := m.convertResources() + report.Outputs = outputs + + // 4. Calculate final statistics from the handler's tracked statuses + for _, status := range handler.ResourceStatuses() { + switch status { + case StatusSuccess: + report.SuccessCount++ + case StatusSkipped: + report.SkippedCount++ + case StatusWarning: + report.WarningCount++ + case StatusFailed: + report.FailedCount++ + } + } + + return report, nil +} + +// PrintSummary formats and writes the standardized migration report summary. +func (m *Migrator) PrintSummary(r *MigrationReport) { + fmt.Fprintln(m.Stderr, "\n=========================================") + fmt.Fprintln(m.Stderr, "Migration Complete Summary:") + fmt.Fprintf(m.Stderr, " Successfully Migrated: %d\n", r.SuccessCount) + fmt.Fprintf(m.Stderr, " Migrated with Warnings: %d\n", r.WarningCount) + fmt.Fprintf(m.Stderr, " Skipped (Unsupported): %d\n", r.SkippedCount) + fmt.Fprintf(m.Stderr, " Failed: %d\n", r.FailedCount) + fmt.Fprintln(m.Stderr, "=========================================") +} + +// WriteOutputs serializes and writes the converted manifests to the migrator's Stdout stream +// in standard Kubernetes multi-document YAML format (separated by "---"). +func (m *Migrator) WriteOutputs(outputs []*unstructured.Unstructured) error { + var buf strings.Builder + for i, out := range outputs { + if out == nil || out.Object == nil { + return fmt.Errorf("internal error: found nil resource or uninitialized object in outputs at index %d", i) + } + + yamlOut, err := yaml.Marshal(out) + if err != nil { + return err + } + if i > 0 { + if _, err := fmt.Fprintln(&buf, "---"); err != nil { + return fmt.Errorf("failed to write document separator: %w", err) + } + } + if _, err := buf.Write(yamlOut); err != nil { + return fmt.Errorf("failed to write output: %w", err) + } + } + if _, err := io.WriteString(m.Stdout, buf.String()); err != nil { + return fmt.Errorf("failed to write output to destination: %w", err) + } + return nil +} + +// isRelevantKind returns true if the input resource Kind is either a target +// resource with a registered converter, or a known dependency. +func (m *Migrator) isRelevantKind(kind string) bool { + switch kind { + case "Service", "ConfigMap", "Secret": + return true + } + _, registered := m.converters[kind] + return registered +} + +// parseInputs reads files, directories, or stdin and loads them into the cache. +func (m *Migrator) parseInputs(path string) error { + // 1. Handle Stdin Strm + if path == "-" { + if err := m.parseYAMLStream(m.Stdin); err != nil { + // Log and track the error + m.logger.Error("Skipping stdin due to parse error", + slog.String("file", "-"), + slog.Any("error", err), + ) + } + return nil + } + + // 2. Resolve system errors + info, err := os.Stat(path) + if err != nil { + return err + } + + // 3. Handle Single Direct File + if !info.IsDir() { + if err := m.parseFile(path); err != nil { + // Log and track the error + m.logger.Error("Skipping file due to parse error", + slog.String("file", path), + slog.Any("error", err), + ) + } + return nil + } + + // 4. Handle Directory Walk + return filepath.WalkDir(path, func(fp string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if fp == path { + return nil + } + // Skip hidden subdirectories encountered during the walk + if d.Name() != "." && d.Name() != ".." && strings.HasPrefix(d.Name(), ".") { + return filepath.SkipDir + } + return nil + } + + // Skip hidden files encountered during the walk (e.g. .yamllint.yaml) + if strings.HasPrefix(d.Name(), ".") { + return nil + } + + ext := strings.ToLower(filepath.Ext(fp)) + if ext == ".yaml" || ext == ".yml" { + if err := m.parseFile(fp); err != nil { + // Log and track the error + m.logger.Error("Skipping file due to parse error", + slog.String("file", fp), + slog.Any("error", err), + ) + } + } + return nil + }) +} + +func (m *Migrator) parseFile(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return m.parseYAMLStream(f) +} + +func (m *Migrator) parseYAMLStream(r io.Reader) error { + decoder := k8syaml.NewYAMLOrJSONDecoder(r, 4096) + for { + var u unstructured.Unstructured + if err := decoder.Decode(&u); err != nil { + if err == io.EOF { + break + } + return err + } + + if err := m.processUnstructured(&u); err != nil { + return err + } + } + return nil +} + +// processUnstructured processes a single unstructured resource. +// If the resource is a List, it recursively processes all nested items. +func (m *Migrator) processUnstructured(u *unstructured.Unstructured) error { + if u.IsList() { + return u.EachListItem(func(obj runtime.Object) error { + nested, ok := obj.(*unstructured.Unstructured) + if !ok { + return errors.New("internal error: failed to cast list item to unstructured") + } + return m.processUnstructured(nested) + }) + } + + apiVersion := u.GetAPIVersion() + kind := u.GetKind() + name := u.GetName() + + // 1. If it's not a resource we care about, skip it. + if !m.isRelevantKind(kind) { + // If it's a Prometheus Operator resource, log a SKIPPED milestone first. + m.logger.Info("Skipping unsupported Prometheus Operator resource", + slog.String("migration_status", "skipped"), + slog.String("apiVersion", apiVersion), + slog.String("kind", kind), + slog.String("namespace", u.GetNamespace()), + slog.String("name", name), + ) + return nil + } + + // 2. Since this IS a resource we care about, it must be well-formed. + if apiVersion == "" || kind == "" || name == "" { + return fmt.Errorf("malformed resource: apiVersion, kind, and metadata.name must all be specified (got apiVersion=%q, kind=%q, name=%q)", apiVersion, kind, name) + } + + if err := m.cache.Add(u); err != nil { + return fmt.Errorf("failed to cache resource: %w", err) + } + return nil +} + +func (m *Migrator) convertResources() []*unstructured.Unstructured { + var allOutputs []*unstructured.Unstructured + ctx := context.Background() + + kinds := slices.AppendSeq(make([]string, 0, len(m.cache.resources)), maps.Keys(m.cache.resources)) + slices.Sort(kinds) + + for _, kind := range kinds { + nsMap := m.cache.resources[kind] + converter, registered := m.converters[kind] + if !registered { + continue + } + + keys := slices.AppendSeq(make([]string, 0, len(nsMap)), maps.Keys(nsMap)) + slices.Sort(keys) + + for _, key := range keys { + res := nsMap[key].DeepCopy() + + // Create the resource logger + resourceLogger := m.logger.With( + slog.String("kind", kind), + slog.String("namespace", res.GetNamespace()), + slog.String("name", res.GetName()), + ) + + outputs, err := converter.Convert(ctx, resourceLogger, res, m.cache) + + if err != nil { + resourceLogger.Error(err.Error()) + continue + } + + allOutputs = append(allOutputs, outputs...) + + resourceLogger.Debug("Converted successfully") + } + } + return allOutputs +} diff --git a/pkg/migrate/migrate_test.go b/pkg/migrate/migrate_test.go new file mode 100644 index 0000000000..c8df0d2deb --- /dev/null +++ b/pkg/migrate/migrate_test.go @@ -0,0 +1,417 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "bytes" + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// TestPodMonitorConverter implements ResourceConverter for testing. +type TestPodMonitorConverter struct { + calls int +} + +func (t *TestPodMonitorConverter) ImportKey() string { + return "PodMonitor" +} + +func (t *TestPodMonitorConverter) Convert(_ context.Context, logger *slog.Logger, unstruct *unstructured.Unstructured, cache *ResourceCache) ([]*unstructured.Unstructured, error) { + t.calls++ + + _, found := cache.Get("Service", unstruct.GetNamespace(), "backing-service") + + if !found { + logger.Warn("backing-service not found in cache") + } else { + logger.Info("Successfully resolved backing-service") + } + + out := &unstructured.Unstructured{} + out.SetGroupVersionKind(unstruct.GroupVersionKind()) + out.SetKind("TranslatedDummy") + out.SetName("translated-" + unstruct.GetName()) + out.SetNamespace(unstruct.GetNamespace()) + + return []*unstructured.Unstructured{out}, nil +} + +func TestMigratorCacheAndExtensibility(t *testing.T) { + tmpDir := t.TempDir() + + yamlContent := ` +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: my-monitor + namespace: default +spec: + foo: bar +--- +apiVersion: v1 +kind: Service +metadata: + name: backing-service + namespace: default +spec: + ports: + - port: 80 +` + inputFilePath := filepath.Join(tmpDir, "input.yaml") + if err := os.WriteFile(inputFilePath, []byte(yamlContent), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + migrator := NewMigrator() + var stdoutBuf, stderrBuf bytes.Buffer + migrator.Stdout = &stdoutBuf + migrator.Stderr = &stderrBuf + + testConv := &TestPodMonitorConverter{} + migrator.RegisterConverter(testConv) + + // Run migration + report, err := migrator.Run(inputFilePath) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + if testConv.calls != 1 { + t.Errorf("expected TestPodMonitorConverter to be called 1 time, got %d", testConv.calls) + } + + // Verify report stats + if report.SuccessCount != 1 { + t.Errorf("expected SuccessCount to be 1, got %d", report.SuccessCount) + } + if report.WarningCount != 0 { + t.Errorf("expected WarningCount to be 0, got %d", report.WarningCount) + } + if report.SkippedCount != 0 { + t.Errorf("expected SkippedCount to be 0, got %d", report.SkippedCount) + } + + // Verify in-memory converted outputs + if len(report.Outputs) != 1 { + t.Errorf("expected 1 output resource, got %d", len(report.Outputs)) + } else { + out := report.Outputs[0] + if out.GetKind() != "TranslatedDummy" { + t.Errorf("expected output kind 'TranslatedDummy', got %q", out.GetKind()) + } + if out.GetName() != "translated-my-monitor" { + t.Errorf("expected output name 'translated-my-monitor', got %q", out.GetName()) + } + } + + stderrLogs := stderrBuf.String() + if !strings.Contains(stderrLogs, "[INFO] [PodMonitor:default/my-monitor] Successfully resolved backing-service") { + t.Errorf("expected formatted INFO log in Stderr, got: %q", stderrLogs) + } + if !strings.Contains(stderrLogs, "[SUCCESS] [PodMonitor:default/my-monitor] Converted successfully") { + t.Errorf("expected formatted SUCCESS log in Stderr, got: %q", stderrLogs) + } +} + +func TestResourceCacheNamespaceScoping(t *testing.T) { + cache := NewResourceCache() + + omittedNsRes := &unstructured.Unstructured{} + omittedNsRes.SetAPIVersion("monitoring.coreos.com/v1") + omittedNsRes.SetKind("PodMonitor") + omittedNsRes.SetName("my-monitor-omitted") + omittedNsRes.SetNamespace("") + + if err := cache.Add(omittedNsRes); err != nil { + t.Fatalf("Add failed: %v", err) + } + + if _, found := cache.Get("PodMonitor", "", "my-monitor-omitted"); !found { + t.Error("expected namespaced resource with omitted namespace to be found under empty namespace") + } + + nsARes := &unstructured.Unstructured{} + nsARes.SetAPIVersion("monitoring.coreos.com/v1") + nsARes.SetKind("PodMonitor") + nsARes.SetName("common-name") + nsARes.SetNamespace("namespace-a") + + if err := cache.Add(nsARes); err != nil { + t.Fatalf("Add failed: %v", err) + } + + if _, found := cache.Get("PodMonitor", "namespace-b", "common-name"); found { + t.Error("expected strict namespace isolation; found resource from namespace-a when querying namespace-b") + } + + res, found := cache.Get("PodMonitor", "namespace-a", "common-name") + if !found { + t.Fatal("expected to find resource in namespace-a") + } + if res.GetNamespace() != "namespace-a" { + t.Errorf("expected found resource to have namespace 'namespace-a', got %q", res.GetNamespace()) + } +} + +func TestMigratorMalformedInput(t *testing.T) { + tmpDir := t.TempDir() + + // YAML resource with a Kind but completely missing metadata.name + malformedYAML := ` +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + namespace: default +spec: + selector: + matchLabels: + app: my-app +` + inputFilePath := filepath.Join(tmpDir, "bad_resource.yaml") + if err := os.WriteFile(inputFilePath, []byte(malformedYAML), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + migrator := NewMigrator() + migrator.RegisterConverter(&TestPodMonitorConverter{}) + var stdoutBuf, stderrBuf bytes.Buffer + migrator.Stdout = &stdoutBuf + migrator.Stderr = &stderrBuf + + // Run migration on the directory containing the malformed file + report, err := migrator.Run(tmpDir) + if err != nil { + t.Fatalf("Run should not return a fatal error for directory walks, got: %v", err) + } + + // Verify that the file parse error was caught and counted as a failure. + if report.FailedCount != 1 { + t.Errorf("expected FailedCount to be 1, got %d", report.FailedCount) + } + if report.SuccessCount != 0 { + t.Errorf("expected SuccessCount to be 0, got %d", report.SuccessCount) + } + if report.WarningCount != 0 { + t.Errorf("expected WarningCount to be 0, got %d", report.WarningCount) + } + if report.SkippedCount != 0 { + t.Errorf("expected SkippedCount to be 0, got %d", report.SkippedCount) + } + + // Verify that a [ERROR] log was printed to Stderr showing the file path and exact parse error + stderrLogs := stderrBuf.String() + if !strings.Contains(stderrLogs, "[ERROR] ["+inputFilePath+"] Skipping file due to parse error") { + t.Errorf("expected formatted [ERROR] log in Stderr, got: %q", stderrLogs) + } + if !strings.Contains(stderrLogs, "malformed resource: apiVersion, kind, and metadata.name must all be specified") { + t.Errorf("expected underlying parse error in Stderr, got: %q", stderrLogs) + } +} + +func TestMigratorSkippedResource(t *testing.T) { + tmpDir := t.TempDir() + + // Unsupported Prometheus Operator resource kind (Alertmanager) + skippedYAML := ` +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: my-alertmanager +spec: + replicas: 3 +` + inputFilePath := filepath.Join(tmpDir, "skipped_resource.yaml") + if err := os.WriteFile(inputFilePath, []byte(skippedYAML), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + migrator := NewMigrator() + migrator.RegisterConverter(&TestPodMonitorConverter{}) + var stdoutBuf, stderrBuf bytes.Buffer + migrator.Stdout = &stdoutBuf + migrator.Stderr = &stderrBuf + + // Run migration on the directory containing the skipped file + report, err := migrator.Run(tmpDir) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + // Verify report stats + if report.SkippedCount != 1 { + t.Errorf("expected SkippedCount to be 1, got %d", report.SkippedCount) + } + if report.SuccessCount != 0 { + t.Errorf("expected SuccessCount to be 0, got %d", report.SuccessCount) + } + if report.WarningCount != 0 { + t.Errorf("expected WarningCount to be 0, got %d", report.WarningCount) + } + if report.FailedCount != 0 { + t.Errorf("expected FailedCount to be 0, got %d", report.FailedCount) + } + + // Verify that a [SKIPPED] log was printed to Stderr showing the resource details + stderrLogs := stderrBuf.String() + if !strings.Contains(stderrLogs, "[SKIPPED] [Alertmanager:my-alertmanager] Skipping unsupported Prometheus Operator resource") { + t.Errorf("expected formatted [SKIPPED] log in Stderr, got: %q", stderrLogs) + } +} + +func TestMigratorMultipleInputs(t *testing.T) { + tmpDir := t.TempDir() + + // 1. Write a Service manifest to a separate file + serviceYAML := ` +apiVersion: v1 +kind: Service +metadata: + name: backing-service + namespace: default +spec: + ports: + - port: 80 +` + servicePath := filepath.Join(tmpDir, "service.yaml") + if err := os.WriteFile(servicePath, []byte(serviceYAML), 0644); err != nil { + t.Fatalf("failed to write service file: %v", err) + } + + // 2. Write a PodMonitor manifest referencing that service to a separate file + podMonitorYAML := ` +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: my-monitor + namespace: default +spec: + foo: bar +` + podMonitorPath := filepath.Join(tmpDir, "podmonitor.yaml") + if err := os.WriteFile(podMonitorPath, []byte(podMonitorYAML), 0644); err != nil { + t.Fatalf("failed to write podmonitor file: %v", err) + } + + migrator := NewMigrator() + var stdoutBuf, stderrBuf bytes.Buffer + migrator.Stdout = &stdoutBuf + migrator.Stderr = &stderrBuf + + testConv := &TestPodMonitorConverter{} + migrator.RegisterConverter(testConv) + + // Run migration passing both files explicitly! + report, err := migrator.Run(servicePath, podMonitorPath) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + if testConv.calls != 1 { + t.Errorf("expected TestPodMonitorConverter to be called 1 time, got %d", testConv.calls) + } + + // Verify report stats + if report.SuccessCount != 1 { + t.Errorf("expected SuccessCount to be 1, got %d", report.SuccessCount) + } + if report.WarningCount != 0 { + t.Errorf("expected WarningCount to be 0, got %d", report.WarningCount) + } + if report.SkippedCount != 0 { + t.Errorf("expected SkippedCount to be 0, got %d", report.SkippedCount) + } + if report.FailedCount != 0 { + t.Errorf("expected FailedCount to be 0, got %d", report.FailedCount) + } + + // Verify that the reference was successfully resolved across the separate files! + stderrLogs := stderrBuf.String() + if !strings.Contains(stderrLogs, "[INFO] [PodMonitor:default/my-monitor] Successfully resolved backing-service") { + t.Errorf("expected reference to be successfully resolved, got logs: %q", stderrLogs) + } +} + +func TestMigratorPipedList(t *testing.T) { + // A standard v1.List containing a Service and a PodMonitor in its items array + listYAML := ` +apiVersion: v1 +kind: List +metadata: + resourceVersion: "" +items: +- apiVersion: v1 + kind: Service + metadata: + name: backing-service + namespace: default + spec: + ports: + - port: 80 +- apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: my-monitor + namespace: default + spec: + foo: bar +` + migrator := NewMigrator() + var stdoutBuf, stderrBuf bytes.Buffer + migrator.Stdout = &stdoutBuf + migrator.Stderr = &stderrBuf + + // Pipe the list YAML buffer directly into Stdin! + migrator.Stdin = strings.NewReader(listYAML) + + testConv := &TestPodMonitorConverter{} + migrator.RegisterConverter(testConv) + + // Run migration using "-" (Stdin) + report, err := migrator.Run("-") + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + if testConv.calls != 1 { + t.Errorf("expected TestPodMonitorConverter to be called 1 time, got %d", testConv.calls) + } + + // Verify report stats + if report.SuccessCount != 1 { + t.Errorf("expected SuccessCount to be 1, got %d", report.SuccessCount) + } + if report.WarningCount != 0 { + t.Errorf("expected WarningCount to be 0, got %d", report.WarningCount) + } + if report.SkippedCount != 0 { + t.Errorf("expected SkippedCount to be 0, got %d", report.SkippedCount) + } + if report.FailedCount != 0 { + t.Errorf("expected FailedCount to be 0, got %d", report.FailedCount) + } + + // Verify that the PodMonitor resolved the Service successfully inside the list! + stderrLogs := stderrBuf.String() + if !strings.Contains(stderrLogs, "[INFO] [PodMonitor:default/my-monitor] Successfully resolved backing-service") { + t.Errorf("expected reference to be successfully resolved inside list, got logs: %q", stderrLogs) + } +} diff --git a/pkg/migrate/types.go b/pkg/migrate/types.go new file mode 100644 index 0000000000..32f6462049 --- /dev/null +++ b/pkg/migrate/types.go @@ -0,0 +1,99 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ResourceConverter defines the interface for converting a specific Prometheus Operator resource kind. +type ResourceConverter interface { + // ImportKey returns the Kind of the resource this converter handles (e.g., "PodMonitor"). + ImportKey() string + // Convert translates the input unstructured resource to one or more GMP resources. + Convert(ctx context.Context, logger *slog.Logger, unstruct *unstructured.Unstructured, cache *ResourceCache) (outputs []*unstructured.Unstructured, err error) +} + +// ResourceCache stores parsed Kubernetes resources for cross-resource resolution. +type ResourceCache struct { + // Map of Kind -> Namespace/Name -> Resource + resources map[string]map[string]*unstructured.Unstructured +} + +// NewResourceCache creates a new initialized ResourceCache. +func NewResourceCache() *ResourceCache { + return &ResourceCache{ + resources: make(map[string]map[string]*unstructured.Unstructured), + } +} + +// Add adds a resource to the cache, returning an error if inputs are invalid. +func (c *ResourceCache) Add(u *unstructured.Unstructured) error { + if c == nil { + return errors.New("cannot add to nil ResourceCache") + } + if u == nil { + return errors.New("cannot add nil resource to cache") + } + + name := u.GetName() + if name == "" { + return errors.New("cannot add resource with empty name to cache") + } + kind := u.GetKind() + if kind == "" { + return errors.New("cannot add resource with empty kind to cache") + } + apiVersion := u.GetAPIVersion() + if apiVersion == "" { + return errors.New("cannot add resource with empty apiVersion to cache") + } + + if c.resources == nil { + c.resources = make(map[string]map[string]*unstructured.Unstructured) + } + + if _, ok := c.resources[kind]; !ok { + c.resources[kind] = make(map[string]*unstructured.Unstructured) + } + + ns := u.GetNamespace() + + key := fmt.Sprintf("%s/%s", ns, name) + if _, exists := c.resources[kind][key]; exists { + return fmt.Errorf("duplicate resource %s/%s found in cache", kind, key) + } + c.resources[kind][key] = u + return nil +} + +// Get retrieves a resource from the cache by kind, namespace, and name. +func (c *ResourceCache) Get(kind, namespace, name string) (*unstructured.Unstructured, bool) { + if c == nil || c.resources == nil { + return nil, false + } + nsMap, ok := c.resources[kind] + if !ok { + return nil, false + } + key := fmt.Sprintf("%s/%s", namespace, name) + r, ok := nsMap[key] + return r, ok +}