refactor: optimize Spanner batching and standardize OpenTelemetry deployment

.github/dependabot.yml

@@ -78,11 +78,6 @@ updates:
           - '>=23'
           - '<24'
 
-  - package-ecosystem: 'docker'
-    directory: '/otel'
-    schedule:
-      interval: 'weekly'
-
   - package-ecosystem: 'docker'
     directory: '/.dev/auth'
     schedule:

backend/cmd/server/main.go

@@ -59,6 +59,7 @@ func main() {
 		datastoreDB = &value
 	}
 	projectID := os.Getenv("PROJECT_ID")
+	slog.InfoContext(context.TODO(), "BOOT: Creating Datastore client...")
 	fs, err := gds.NewDatastoreClient(projectID, datastoreDB)
 	if err != nil {
 		slog.ErrorContext(context.TODO(), "failed to create datastore client", "error", err.Error())
@@ -69,6 +70,7 @@ func main() {
 
 	spannerDB := os.Getenv("SPANNER_DATABASE")
 	spannerInstance := os.Getenv("SPANNER_INSTANCE")
+	slog.InfoContext(ctx, "BOOT: Creating Spanner client...")
 	spannerClient, err := gcpspanner.NewSpannerClient(projectID, spannerInstance, spannerDB)
 	if err != nil {
 		slog.ErrorContext(ctx, "failed to create spanner client", "error", err.Error())
@@ -106,6 +108,7 @@ func main() {
 	slog.InfoContext(ctx, "cache settings", "duration", cacheTTL, "prefix", cacheKeyPrefix,
 		"aggregatedFeaturesStatsTTL", aggregatedFeaturesStatsTTL)
 
+	slog.InfoContext(ctx, "BOOT: Creating Valkey cache...")
 	cache, err := valkeycache.NewValkeyDataCache[string, []byte](
 		cacheKeyPrefix,
 		valkeyHost,
@@ -165,25 +168,22 @@ func main() {
 			}),
 	}
 
-	if os.Getenv("OTEL_SERVICE_NAME") != "" {
-		slog.InfoContext(ctx, "opentelemetry settings detected.")
-		otelProjectID := os.Getenv("OTEL_GCP_PROJECT_ID")
-		if otelProjectID == "" {
-			slog.ErrorContext(ctx, "missing project id for opentelemetry")
-			os.Exit(1)
-		}
-		shutdown, err := opentelemetry.SetupOpenTelemetry(ctx, otelProjectID)
-		if err != nil {
-			slog.ErrorContext(ctx, "failed to setup opentelemetry", "error", err.Error())
-			os.Exit(1)
-		}
+	slog.InfoContext(ctx, "BOOT: Running OTel setup...")
+	shutdown, err := opentelemetry.MaybeSetup(ctx)
+	if err != nil {
+		slog.ErrorContext(ctx, "failed to setup opentelemetry", "error", err.Error())
+		os.Exit(1)
+	}
+	if shutdown != nil {
 		defer func() {
-			err := shutdown(ctx)
-			if err != nil {
-				slog.ErrorContext(ctx, "unable to shutdown opentelemetry")
+			if err := shutdown(ctx); err != nil {
+				slog.ErrorContext(ctx, "unable to shutdown opentelemetry", "error", err)
 			}
 		}()
-		// Prepend the opentelemtry middleware
+	}
+
+	if os.Getenv("OTEL_SERVICE_NAME") != "" {
+		// Prepend the opentelemetry middleware
 		preRequestMiddlewares = slices.Insert(preRequestMiddlewares, 0, opentelemetry.NewOpenTelemetryChiMiddleware())
 	}
 

infra/.envs/prod.tfvars

@@ -5,6 +5,7 @@ projects = {
 }
 project_name             = "web-compass-prod"
 spanner_processing_units = 400
+spanner_edition          = "ENTERPRISE_PLUS"
 deletion_protection      = true
 secret_ids = {
   github_token = "jamestestGHtoken-readonly"

infra/.envs/staging.tfvars

@@ -5,6 +5,7 @@ projects = {
 }
 project_name             = "web-compass-staging"
 spanner_processing_units = 400
+spanner_edition          = "STANDARD"
 deletion_protection      = true
 secret_ids = {
   github_token = "jamestestGHtoken-readonly"

infra/backend/service.tf

@@ -46,13 +46,11 @@ resource "docker_registry_image" "backend_remote_image" {
 data "google_project" "host_project" {
 }
 
-module "otel_sidecar" {
-  source = "../modules/otel"
-  providers = {
-    google.project = google.public_project
-  }
-  docker_repository_details = var.docker_repository_details
-  service_account           = google_service_account.backend.member
+resource "google_secret_manager_secret_iam_member" "backend_otel_config_secret_access" {
+  provider  = google.internal_project
+  secret_id = var.otel_config_secret_id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.backend.email}"
 }
 
 resource "google_cloud_run_v2_service" "service" {
@@ -125,7 +123,7 @@ resource "google_cloud_run_v2_service" "service" {
       }
       env {
         name  = "OTEL_EXPORTER_OTLP_ENDPOINT"
-        value = "http://localhost:4318"
+        value = var.otel_collector_endpoint
       }
       env {
         name  = "OTEL_SERVICE_NAME"
@@ -150,7 +148,16 @@ resource "google_cloud_run_v2_service" "service" {
     }
     containers {
       name  = "otel"
-      image = module.otel_sidecar.otel_image
+      image = var.otel_collector_image
+      args  = ["--config=${var.otel_collector_config_mount_path}/config.yaml"]
+      env {
+        name  = "OTEL_COLLECTOR_REGION"
+        value = each.key
+      }
+      volume_mounts {
+        name       = "otel-config"
+        mount_path = var.otel_collector_config_mount_path
+      }
       liveness_probe {
         http_get {
           port = 4319
@@ -162,7 +169,7 @@ resource "google_cloud_run_v2_service" "service" {
         timeout_seconds       = 10
       }
       startup_probe {
-        initial_delay_seconds = 0
+        initial_delay_seconds = 5
         timeout_seconds       = 1
         period_seconds        = 3
         failure_threshold     = 10
@@ -178,6 +185,16 @@ resource "google_cloud_run_v2_service" "service" {
       }
       egress = "PRIVATE_RANGES_ONLY"
     }
+    volumes {
+      name = "otel-config"
+      secret {
+        secret = var.otel_config_secret_id
+        items {
+          version = "latest"
+          path    = "config.yaml"
+        }
+      }
+    }
     service_account = google_service_account.backend.email
   }
 
@@ -188,6 +205,7 @@ resource "google_cloud_run_v2_service" "service" {
 
   depends_on = [
     google_project_iam_member.gcp_datastore_user,
+    google_secret_manager_secret_iam_member.backend_otel_config_secret_access,
   ]
 
   deletion_protection = var.deletion_protection
@@ -327,3 +345,29 @@ resource "google_compute_managed_ssl_certificate" "lb_default" {
     domains = var.domains
   }
 }
+
+# --- Telemetry IAM Permissions for Backend Service Account ---
+
+# Grant Cloud Trace Agent role to allow exporting traces directly or via OTel sidecar.
+resource "google_project_iam_member" "backend_trace_agent" {
+  provider = google.public_project
+  project  = var.projects.public
+  role     = "roles/cloudtrace.agent"
+  member   = "serviceAccount:${google_service_account.backend.email}"
+}
+
+# Grant Monitoring Metric Writer role to allow exporting metrics.
+resource "google_project_iam_member" "backend_metric_writer" {
+  provider = google.public_project
+  project  = var.projects.public
+  role     = "roles/monitoring.metricWriter"
+  member   = "serviceAccount:${google_service_account.backend.email}"
+}
+
+# Grant Logging Log Writer role to allow exporting structured logs.
+resource "google_project_iam_member" "backend_log_writer" {
+  provider = google.public_project
+  project  = var.projects.public
+  role     = "roles/logging.logWriter"
+  member   = "serviceAccount:${google_service_account.backend.email}"
+}

infra/backend/variables.tf

@@ -110,3 +110,23 @@ variable "backend_api_url" {
 
 variable "pubsub_project_id" { type = string }
 variable "ingestion_topic_id" { type = string }
+
+variable "otel_config_secret_id" {
+  type        = string
+  description = "The Secret Manager secret ID containing the OTel collector configuration"
+}
+
+variable "otel_collector_image" {
+  type        = string
+  description = "The container image to use for the OTel collector sidecar"
+}
+
+variable "otel_collector_config_mount_path" {
+  type        = string
+  description = "The volume mount path for the OTel collector configuration"
+}
+
+variable "otel_collector_endpoint" {
+  type        = string
+  description = "The endpoint for the application to export OTLP metrics/traces to the local collector"
+}

infra/ingestion/variables.tf

@@ -94,3 +94,28 @@ variable "notification_channel_ids" {
   description = "A list of notification channel ids to send alerts to."
   type        = list(string)
 }
+
+variable "otel_config_secret_id" {
+  type        = string
+  description = "The Secret Manager secret ID containing the OTel collector configuration"
+}
+
+variable "otel_project_id" {
+  type        = string
+  description = "The GCP project ID where telemetry traces/metrics will be exported"
+}
+
+variable "otel_collector_image" {
+  type        = string
+  description = "The container image to use for the OTel collector sidecar"
+}
+
+variable "otel_collector_config_mount_path" {
+  type        = string
+  description = "The volume mount path for the OTel collector configuration"
+}
+
+variable "otel_collector_endpoint" {
+  type        = string
+  description = "The endpoint for the application to export OTLP metrics/traces to the local collector"
+}

infra/ingestion/workflows.tf

@@ -59,6 +59,11 @@ module "web_features_workflow" {
       value = "262980h" # 30 years in hours (365.25*30*24)
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "wpt_workflow" {
@@ -105,9 +110,14 @@ module "wpt_workflow" {
     },
     {
       name  = "DATA_WINDOW_DURATION"
-      value = "17520h" # 2 years
+      value = "720h" # 30 days
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "bcd_workflow" {
@@ -144,6 +154,11 @@ module "bcd_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "chromium_enum_workflow" {
@@ -180,6 +195,11 @@ module "chromium_enum_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "uma_export_workflow" {
@@ -216,6 +236,11 @@ module "uma_export_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "developer_signals_workflow" {
@@ -252,6 +277,11 @@ module "developer_signals_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "web_features_mapping_workflow" {
@@ -288,4 +318,9 @@ module "web_features_mapping_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }

infra/main.tf

@@ -65,6 +65,7 @@ module "storage" {
   spanner_region_id         = local.spanner_repository_region
   datastore_region_id       = var.datastore_region_id
   spanner_processing_units  = var.spanner_processing_units
+  spanner_edition           = var.spanner_edition
   docker_repository_region  = local.docker_repository_region
   projects                  = var.projects
   depends_on                = [module.services]
@@ -96,6 +97,11 @@ module "ingestion" {
   developer_signals_region_schedules    = var.developer_signals_region_schedules
   web_features_mapping_region_schedules = var.web_features_mapping_region_schedules
   notification_channel_ids              = var.notification_channel_ids
+  otel_config_secret_id                 = google_secret_manager_secret.otel_config.id
+  otel_project_id                       = var.projects.internal
+  otel_collector_image                  = local.otel_collector_image
+  otel_collector_config_mount_path      = local.otel_collector_config_mount_path
+  otel_collector_endpoint               = local.otel_collector_endpoint
 }
 
 module "backend" {
@@ -124,8 +130,12 @@ module "backend" {
   firebase_settings = {
     tenant_id = module.auth.tenant_id
   }
-  pubsub_project_id  = var.projects.internal
-  ingestion_topic_id = module.pubsub.ingestion_topic_id
+  pubsub_project_id                = var.projects.internal
+  ingestion_topic_id               = module.pubsub.ingestion_topic_id
+  otel_config_secret_id            = google_secret_manager_secret.otel_config.id
+  otel_collector_image             = local.otel_collector_image
+  otel_collector_config_mount_path = local.otel_collector_config_mount_path
+  otel_collector_endpoint          = local.otel_collector_endpoint
 }
 
 module "frontend" {
@@ -205,6 +215,11 @@ module "workers" {
 
   chime_details = var.chime_details
 
-  email_service_account_email = var.email_service_account_email
-  deletion_protection         = var.deletion_protection
+  email_service_account_email      = var.email_service_account_email
+  deletion_protection              = var.deletion_protection
+  otel_config_secret_id            = google_secret_manager_secret.otel_config.id
+  otel_project_id                  = var.projects.internal
+  otel_collector_image             = local.otel_collector_image
+  otel_collector_config_mount_path = local.otel_collector_config_mount_path
+  otel_collector_endpoint          = local.otel_collector_endpoint
 }