Feat: export GCP metrics via OTLP to telemetry.googleapis.com instead of the Cloud Monitoring exporter

core/package.json

@@ -41,17 +41,18 @@
   },
   "dependencies": {
     "@a2a-js/sdk": "^0.3.10",
-    "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.21.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^3.0.0",
     "@google-cloud/storage": "^7.17.1",
     "@google-cloud/vertexai": "^1.12.0",
     "@google/genai": "^2.9.0",
+    "@grpc/grpc-js": "^1.14.4",
     "@mikro-orm/core": "^6.6.10",
     "@mikro-orm/reflection": "^6.6.6",
     "@modelcontextprotocol/sdk": "^1.26.0",
     "@opentelemetry/api": "1.9.0",
     "@opentelemetry/api-logs": "^0.205.0",
     "@opentelemetry/exporter-logs-otlp-http": "^0.205.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.205.0",
     "@opentelemetry/exporter-metrics-otlp-http": "^0.205.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.205.0",
     "@opentelemetry/resource-detector-gcp": "^0.40.0",

core/src/telemetry/google_cloud.ts

@@ -4,13 +4,27 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import {MetricExporter} from '@google-cloud/opentelemetry-cloud-monitoring-exporter';
 import {TraceExporter} from '@google-cloud/opentelemetry-cloud-trace-exporter';
+import {
+  ChannelCredentials,
+  credentials as grpcCredentials,
+  Metadata,
+} from '@grpc/grpc-js';
+import {OTLPMetricExporter} from '@opentelemetry/exporter-metrics-otlp-grpc';
 import {gcpDetector} from '@opentelemetry/resource-detector-gcp';
-import {detectResources, Resource} from '@opentelemetry/resources';
-import {PeriodicExportingMetricReader} from '@opentelemetry/sdk-metrics';
+import {
+  detectResources,
+  envDetector,
+  Resource,
+  resourceFromAttributes,
+  serviceInstanceIdDetector,
+} from '@opentelemetry/resources';
+import {
+  MetricReader,
+  PeriodicExportingMetricReader,
+} from '@opentelemetry/sdk-metrics';
 import {BatchSpanProcessor} from '@opentelemetry/sdk-trace-base';
-import {GoogleAuth} from 'google-auth-library';
+import {AuthClient, GoogleAuth} from 'google-auth-library';
 
 import {logger} from '../utils/logger.js';
 
@@ -20,16 +34,82 @@ const GCP_PROJECT_ERROR_MESSAGE =
   'Cannot determine GCP Project. OTel GCP Exporters cannot be set up. ' +
   'Please make sure to log into correct GCP Project.';
 
-async function getGcpProjectId(): Promise<string | undefined> {
+const GCP_CREDENTIALS_ERROR_MESSAGE =
+  'Cannot obtain Application Default Credentials. OTel GCP metric export is ' +
+  'disabled. Please run `gcloud auth application-default login` or attach a ' +
+  'service account to enable it.';
+
+/** OTLP endpoint of the Google Cloud Telemetry API. */
+const TELEMETRY_ENDPOINT = 'https://telemetry.googleapis.com';
+
+/** Resource attribute the Telemetry API routes ingested metrics on. */
+const GCP_PROJECT_ID_ATTRIBUTE = 'gcp.project_id';
+
+/** Cloud Monitoring rejects sample periods below five seconds. */
+const METRIC_EXPORT_INTERVAL_MS = 5000;
+
+/**
+ * Resolves the GCP project from Application Default Credentials.
+ *
+ * Callers that assemble the telemetry pipeline themselves need this to build
+ * the resource, because the Telemetry API takes the destination project as a
+ * resource attribute rather than an exporter argument.
+ *
+ * @param auth credentials to resolve the project from, defaulting to
+ *   Application Default Credentials.
+ * @returns the project id, or undefined when it cannot be determined.
+ */
+export async function getGcpProjectId(
+  auth: GoogleAuth = new GoogleAuth(),
+): Promise<string | undefined> {
   try {
-    const auth = new GoogleAuth();
     const projectId = await auth.getProjectId();
     return projectId || undefined;
   } catch (_e: unknown) {
     return undefined;
   }
 }
 
+/**
+ * Builds the channel credentials the OTLP exporter authenticates each export
+ * RPC with.
+ *
+ * Uses a metadata generator rather than `createFromGoogleCredential`, which
+ * reads the client's headers with `Object.keys`: `google-auth-library` v10
+ * resolves `getRequestHeaders` to a WHATWG `Headers` instance, whose own
+ * enumerable keys are empty, so that path would send unauthenticated requests.
+ */
+function createChannelCredentials(authClient: AuthClient): ChannelCredentials {
+  const callCredentials = grpcCredentials.createFromMetadataGenerator(
+    (options, callback) => {
+      authClient
+        .getRequestHeaders(options.service_url)
+        .then((headers) => {
+          const metadata = new Metadata();
+          headers.forEach((value, key) => metadata.add(key, value));
+          callback(null, metadata);
+        })
+        .catch((e: unknown) => {
+          callback(e instanceof Error ? e : new Error(String(e)));
+        });
+    },
+  );
+  return grpcCredentials.combineChannelCredentials(
+    grpcCredentials.createSsl(),
+    callCredentials,
+  );
+}
+
+function createGcpMetricReader(authClient: AuthClient): MetricReader {
+  return new PeriodicExportingMetricReader({
+    exporter: new OTLPMetricExporter({
+      url: TELEMETRY_ENDPOINT,
+      credentials: createChannelCredentials(authClient),
+    }),
+    exportIntervalMillis: METRIC_EXPORT_INTERVAL_MS,
+  });
+}
+
 export async function getGcpExporters(
   config: OtelExportersConfig = {},
 ): Promise<OTelHooks> {
@@ -39,28 +119,56 @@ export async function getGcpExporters(
     // enableCloudLogging = false,
   } = config;
 
-  const projectId = await getGcpProjectId();
+  const auth = new GoogleAuth();
+  const projectId = await getGcpProjectId(auth);
   if (!projectId) {
     logger.warn(GCP_PROJECT_ERROR_MESSAGE);
     return {};
   }
 
+  const metricReaders: MetricReader[] = [];
+  if (enableMetrics) {
+    const authClient = await auth.getClient().catch(() => undefined);
+    if (authClient) {
+      metricReaders.push(createGcpMetricReader(authClient));
+    } else {
+      logger.warn(GCP_CREDENTIALS_ERROR_MESSAGE);
+    }
+  }
+
   return {
     spanProcessors: enableTracing
       ? [new BatchSpanProcessor(new TraceExporter({projectId}))]
       : [],
-    metricReaders: enableMetrics
-      ? [
-          new PeriodicExportingMetricReader({
-            exporter: new MetricExporter({projectId}),
-            exportIntervalMillis: 5000,
-          }),
-        ]
-      : [],
+    metricReaders,
     logRecordProcessors: [],
   };
 }
 
-export function getGcpResource(): Resource {
-  return detectResources({detectors: [gcpDetector]});
+/**
+ * Returns the OTel resource to install alongside the GCP exporters.
+ *
+ * Attributes detected later override those detected earlier:
+ * 1. `gcp.project_id` from `projectId`. The Telemetry API routes ingested
+ *    metrics on this attribute; without it the export has no destination
+ *    project, so pass the value {@link getGcpProjectId} resolves.
+ * 2. A generated `service.instance.id`, which supplies the `instance` label
+ *    Managed Service for Prometheus requires and rejects points without.
+ * 3. `OTEL_SERVICE_NAME` / `OTEL_RESOURCE_ATTRIBUTES`. This is the only way to
+ *    supply `location` off Google Cloud, where no detector can infer it.
+ * 4. The GCP detector, which fills in the platform, region and zone when ADK
+ *    runs on GCE, GKE or Cloud Run.
+ *
+ * @param projectId project to attribute the telemetry to.
+ */
+export function getGcpResource(projectId?: string): Resource {
+  const detected = detectResources({
+    detectors: [serviceInstanceIdDetector, envDetector, gcpDetector],
+  });
+  if (!projectId) {
+    return detected;
+  }
+  return resourceFromAttributes({
+    [GCP_PROJECT_ID_ATTRIBUTE]: projectId,
+  }).merge(detected);
 }