initial tracing and log ingestion setup

api/v1alpha1/labels.go

@@ -0,0 +1,15 @@
+package v1alpha1
+
+// Observability label and annotation keys.
+// LabelDistro, LabelTarget, LabelArchitecture are defined in catalogimage_types.go.
+const (
+	LabelBuildMode      = "automotive.sdv.cloud.redhat.com/build-mode"
+	LabelTraceID        = "automotive.sdv.cloud.redhat.com/trace-id"
+	LabelImageBuildName = "automotive.sdv.cloud.redhat.com/imagebuild-name"
+	LabelTaskType       = "automotive.sdv.cloud.redhat.com/task-type"
+	LabelWorkspaceName  = "automotive.sdv.cloud.redhat.com/workspace-name"
+	LabelOwner          = "automotive.sdv.cloud.redhat.com/owner"
+
+	AnnotationTraceID     = "automotive.sdv.cloud.redhat.com/trace-id"
+	AnnotationRequestedBy = "automotive.sdv.cloud.redhat.com/requested-by"
+)

api/v1alpha1/operatorconfig_types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"strconv"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -460,6 +462,58 @@ func (c *MonitoringConfig) GetInterval() string {
 	return "30s"
 }
 
+// TracingConfig defines configuration for OpenTelemetry distributed tracing
+type TracingConfig struct {
+	// Enabled determines if the operator should export trace spans
+	// +kubebuilder:default=false
+	Enabled bool `json:"enabled"`
+
+	// Endpoint is the OTLP gRPC collector endpoint (e.g. "otel-collector.namespace.svc:4317").
+	// Falls back to OTEL_EXPORTER_OTLP_ENDPOINT env var when empty.
+	// +optional
+	Endpoint string `json:"endpoint,omitempty"`
+
+	// Insecure disables TLS for the OTLP gRPC connection.
+	// Default: true (plaintext, appropriate for in-cluster collectors).
+	// Set to false when connecting to external/managed tracing backends that require TLS.
+	// +optional
+	// +kubebuilder:default=true
+	Insecure *bool `json:"insecure,omitempty"`
+
+	// SamplingRatio controls the fraction of traces sampled, as a string representation
+	// of a decimal between "0" and "1" (e.g. "0.1" for 10%, "1" for 100%).
+	// Default: "1" (sample everything).
+	// +optional
+	// +kubebuilder:validation:Pattern=`^(0(\.\d+)?|1(\.0+)?)$`
+	SamplingRatio string `json:"samplingRatio,omitempty"`
+}
+
+// IsInsecure returns whether the OTLP connection should use plaintext (default: true)
+func (c *TracingConfig) IsInsecure() bool {
+	if c != nil && c.Insecure != nil {
+		return *c.Insecure
+	}
+	return true
+}
+
+// GetSamplingRatio returns the sampling ratio as a float64, defaulting to 1.0
+func (c *TracingConfig) GetSamplingRatio() float64 {
+	if c != nil && c.SamplingRatio != "" {
+		if v, err := strconv.ParseFloat(c.SamplingRatio, 64); err == nil {
+			return v
+		}
+	}
+	return 1.0
+}
+
+// GetEndpoint returns the configured endpoint or empty string
+func (c *TracingConfig) GetEndpoint() string {
+	if c != nil && c.Endpoint != "" {
+		return c.Endpoint
+	}
+	return ""
+}
+
 // OperatorConfigSpec defines the desired state of OperatorConfig
 type OperatorConfigSpec struct {
 	// OSBuilds defines the configuration for OS build operations
@@ -489,6 +543,10 @@ type OperatorConfigSpec struct {
 	// Monitoring defines configuration for Prometheus metrics collection
 	// +optional
 	Monitoring *MonitoringConfig `json:"monitoring,omitempty"`
+
+	// Tracing defines configuration for OpenTelemetry distributed tracing
+	// +optional
+	Tracing *TracingConfig `json:"tracing,omitempty"`
 }
 
 // OSBuildsConfig defines configuration for OS build operations
@@ -708,6 +766,9 @@ type OperatorConfigStatus struct {
 	// +patchStrategy=merge
 	// +patchMergeKey=type
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// TracingEnabled indicates if tracing is configured and active
+	TracingEnabled bool `json:"tracingEnabled,omitempty"`
 }
 
 // +kubebuilder:object:root=true

cmd/build-api/main.go

@@ -21,6 +21,7 @@ import (
 
 	automotivev1alpha1 "github.com/centos-automotive-suite/automotive-dev-operator/api/v1alpha1"
 	"github.com/centos-automotive-suite/automotive-dev-operator/internal/buildapi"
+	"github.com/centos-automotive-suite/automotive-dev-operator/internal/common/telemetry"
 )
 
 func main() {
@@ -65,8 +66,27 @@ func main() {
 		}
 	}
 
-	// Load API limits from OperatorConfig
-	limits := loadLimitsFromOperatorConfig(*namespace, logger)
+	// Load config from OperatorConfig
+	configNamespace := os.Getenv("WATCH_NAMESPACE")
+	if configNamespace == "" {
+		configNamespace = *namespace
+	}
+	limits, tracingEnabled, tracingEndpoint, tracingSamplingRatio, tracingInsecure := loadFromOperatorConfig(configNamespace, logger)
+
+	shutdownTracing := func(context.Context) error { return nil }
+	if tracingEnabled {
+		var err error
+		shutdownTracing, err = telemetry.InitTracing(context.Background(), "build-api", tracingEndpoint, tracingSamplingRatio, tracingInsecure)
+		if err != nil {
+			slog.Warn("failed to initialize tracing, continuing without it", "error", err)
+			shutdownTracing = func(context.Context) error { return nil }
+		}
+	}
+	defer func() {
+		if err := shutdownTracing(context.Background()); err != nil {
+			slog.Error("failed to shutdown tracing", "error", err)
+		}
+	}()
 
 	slog.Info("starting build-api server",
 		"addr", addr,
@@ -96,21 +116,28 @@ func main() {
 	}
 }
 
-func loadLimitsFromOperatorConfig(namespace string, logger logr.Logger) buildapi.APILimits {
+func loadFromOperatorConfig(namespace string, logger logr.Logger) (buildapi.APILimits, bool, string, float64, bool) {
 	k8sClient, err := createK8sClient()
 	if err != nil {
-		logger.Info("could not create Kubernetes client, using default limits", "error", err)
-		return buildapi.DefaultAPILimits()
+		logger.Info("could not create Kubernetes client, using defaults", "error", err)
+		return buildapi.DefaultAPILimits(), true, "", 1.0, true
 	}
 
 	operatorConfig := &automotivev1alpha1.OperatorConfig{}
 	key := types.NamespacedName{Name: "config", Namespace: namespace}
 	if err := k8sClient.Get(context.Background(), key, operatorConfig); err != nil {
-		logger.Info("could not get OperatorConfig, using default limits", "error", err)
-		return buildapi.DefaultAPILimits()
+		logger.Info("could not get OperatorConfig, using defaults", "error", err)
+		return buildapi.DefaultAPILimits(), true, "", 1.0, true
+	}
+
+	limits := buildapi.LoadLimitsFromConfig(operatorConfig.Spec.BuildAPI)
+
+	if operatorConfig.Spec.Tracing == nil || !operatorConfig.Spec.Tracing.Enabled {
+		return limits, false, "", 1.0, true
 	}
 
-	return buildapi.LoadLimitsFromConfig(operatorConfig.Spec.BuildAPI)
+	return limits, true, operatorConfig.Spec.Tracing.GetEndpoint(),
+		operatorConfig.Spec.Tracing.GetSamplingRatio(), operatorConfig.Spec.Tracing.IsInsecure()
 }
 
 func createK8sClient() (client.Client, error) {

cmd/main.go

@@ -18,6 +18,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -29,8 +30,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -43,6 +46,7 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
 	automotivev1alpha1 "github.com/centos-automotive-suite/automotive-dev-operator/api/v1alpha1"
+	"github.com/centos-automotive-suite/automotive-dev-operator/internal/common/telemetry"
 	"github.com/centos-automotive-suite/automotive-dev-operator/internal/controller/catalogimage"
 	"github.com/centos-automotive-suite/automotive-dev-operator/internal/controller/containerbuild"
 	"github.com/centos-automotive-suite/automotive-dev-operator/internal/controller/image"
@@ -100,7 +104,7 @@ func main() {
 			"'build' runs only ImageBuild/Image/CatalogImage/ContainerBuild controllers, "+
 			"'all' runs all controllers.")
 	opts := zap.Options{
-		Development: true,
+		Development: false,
 	}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
@@ -200,6 +204,23 @@ func main() {
 	// client-go defaults (QPS=5, Burst=10).
 	restConfig.QPS = 20
 	restConfig.Burst = 30
+
+	shutdownTracing := func(context.Context) error { return nil }
+	tracingEnabled, tracingEndpoint, tracingSamplingRatio, tracingInsecure := readTracingConfig(restConfig, scheme, watchNamespace)
+	if tracingEnabled {
+		var err error
+		shutdownTracing, err = telemetry.InitTracing(context.Background(), "automotive-dev-operator", tracingEndpoint, tracingSamplingRatio, tracingInsecure)
+		if err != nil {
+			setupLog.Error(err, "failed to initialize tracing, continuing without it")
+			shutdownTracing = func(context.Context) error { return nil }
+		}
+	}
+	defer func() {
+		if err := shutdownTracing(context.Background()); err != nil {
+			setupLog.Error(err, "failed to shutdown tracing")
+		}
+	}()
+
 	mgr, err := ctrl.NewManager(restConfig, mgrOptions)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
@@ -308,3 +329,25 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func readTracingConfig(restConfig *rest.Config, s *runtime.Scheme, namespace string) (bool, string, float64, bool) {
+	c, err := client.New(restConfig, client.Options{Scheme: s})
+	if err != nil {
+		setupLog.Info("could not create startup client for tracing config, using env vars")
+		return true, "", 1.0, true
+	}
+
+	var cfg automotivev1alpha1.OperatorConfig
+	key := client.ObjectKey{Name: "config", Namespace: namespace}
+	if err := c.Get(context.Background(), key, &cfg); err != nil {
+		setupLog.Info("could not read OperatorConfig for tracing config, using env vars",
+			"name", key.Name, "namespace", key.Namespace)
+		return true, "", 1.0, true
+	}
+
+	if cfg.Spec.Tracing == nil || !cfg.Spec.Tracing.Enabled {
+		return false, "", 1.0, true
+	}
+
+	return true, cfg.Spec.Tracing.GetEndpoint(), cfg.Spec.Tracing.GetSamplingRatio(), cfg.Spec.Tracing.IsInsecure()
+}

config/crd/bases/automotive.sdv.cloud.redhat.com_operatorconfigs.yaml

@@ -875,6 +875,37 @@ spec:
                 required:
                 - enabled
                 type: object
+              tracing:
+                description: Tracing defines configuration for OpenTelemetry distributed
+                  tracing
+                properties:
+                  enabled:
+                    default: false
+                    description: Enabled determines if the operator should export
+                      trace spans
+                    type: boolean
+                  endpoint:
+                    description: |-
+                      Endpoint is the OTLP gRPC collector endpoint (e.g. "otel-collector.namespace.svc:4317").
+                      Falls back to OTEL_EXPORTER_OTLP_ENDPOINT env var when empty.
+                    type: string
+                  insecure:
+                    default: true
+                    description: |-
+                      Insecure disables TLS for the OTLP gRPC connection.
+                      Default: true (plaintext, appropriate for in-cluster collectors).
+                      Set to false when connecting to external/managed tracing backends that require TLS.
+                    type: boolean
+                  samplingRatio:
+                    description: |-
+                      SamplingRatio controls the fraction of traces sampled, as a string representation
+                      of a decimal between "0" and "1" (e.g. "0.1" for 10%, "1" for 100%).
+                      Default: "1" (sample everything).
+                    pattern: ^(0(\.\d+)?|1(\.0+)?)$
+                    type: string
+                required:
+                - enabled
+                type: object
               workspaces:
                 description: Workspaces defines configuration for developer workspaces
                 properties:
@@ -1169,6 +1200,10 @@ spec:
                 description: Phase represents the current phase (Ready, Reconciling,
                   Failed)
                 type: string
+              tracingEnabled:
+                description: TracingEnabled indicates if tracing is configured and
+                  active
+                type: boolean
               userNamespacesSupported:
                 description: |-
                   UserNamespacesSupported indicates if the cluster supports user namespaces in pods

config/manager/kustomization.yaml

@@ -6,5 +6,5 @@ configurations:
 - kustomizeconfig.yaml
 images:
 - name: controller
-  newName: quay.io/rh-sdv-cloud/automotive-dev-operator
-  newTag: v0.1.0
+  newName: example.com/automotive-dev-operator
+  newTag: v0.0.1