Hotfix Soniox fallback keys for staging

cloud/.env.example

@@ -50,6 +50,7 @@ AZURE_SPEECH_KEY=
 
 # Soniox Speech
 SONIOX_API_KEY=
+SONIOX_FALLBACK_API_KEYS=
 
 # =============================================================================
 # LLM Configuration

cloud/issues/108-soniox-fallback-api-keys/spec.md

@@ -0,0 +1,42 @@
+# Spec: Soniox Fallback API Keys
+
+## Environment
+
+Existing:
+
+```bash
+SONIOX_API_KEY=primary-key
+```
+
+New:
+
+```bash
+SONIOX_FALLBACK_API_KEYS=fallback-key-a,fallback-key-b,fallback-key-c
+```
+
+`SONIOX_API_KEY` remains the preferred primary credential. Fallback keys are
+comma-separated, trimmed, and deduplicated. Empty entries are ignored.
+
+## Runtime Behavior
+
+1. New transcription stream creation first tries the primary key when it is not
+   cooling down.
+2. If the primary key is unavailable or stream creation fails with a
+   credential/limit/provider error, stream creation tries fallback keys.
+3. Fallback keys are chosen round-robin among keys that are not cooling down.
+4. No local max-concurrent accounting is used.
+5. Errors are classified into cooldown classes:
+   - concurrent stream limit: very short cooldown
+   - request/rate limit: short cooldown
+   - spend/account quota: long cooldown
+   - invalid key/authentication: disabled for this process
+   - transient/network/server: short cooldown
+6. Logs include credential fingerprints only. Raw API keys must never be logged.
+7. Existing transcription and translation retry behavior remains in place. A
+   retry should create a new stream, which reselects a Soniox key from the pool.
+
+## Non-Goals
+
+- Per-key concurrency env vars.
+- Cross-pod key usage coordination.
+- New external state such as Redis for quota tracking.

cloud/issues/108-soniox-fallback-api-keys/spike.md

@@ -0,0 +1,45 @@
+# Spike: Soniox Fallback API Keys
+
+## Problem
+
+Legacy Cloud v1 currently constructs Soniox transcription and translation
+providers from `SONIOX_API_KEY`. When that Soniox org/key hits an account-level
+limit, every new Soniox transcription or translation stream in production fails
+through the same exhausted credential. The existing provider fallback machinery
+only switches between provider types. It does not support multiple Soniox
+credentials.
+
+## Observed Code Path
+
+- `cloud/packages/cloud/src/services/session/transcription/types.ts`
+  reads `SONIOX_API_KEY` into `DEFAULT_TRANSCRIPTION_CONFIG.soniox.apiKey`.
+- `SonioxTranscriptionProvider` initializes one Soniox SDK client with that key.
+- `TranscriptionManager` creates one Soniox provider for non-China deployments.
+- Stream retry logic retries the same Soniox provider/key after 429, 408, and
+  server errors.
+- `cloud/packages/cloud/src/services/session/translation/types.ts` also reads
+  `SONIOX_API_KEY` into `DEFAULT_TRANSLATION_CONFIG.soniox.apiKey`.
+- `TranslationManager` retries translation streams, but without multiple Soniox
+  credentials it retries the same exhausted key.
+
+## Important Constraint
+
+Do not configure local max-concurrent limits per key. Soniox keys may be shared
+across pods or environments, so a local counter is incomplete and can make a key
+look available when another process already consumed its concurrency quota. The
+source of truth is Soniox accepting or rejecting a stream.
+
+## Failure Classes
+
+- Spend or account quota exhausted: long cooldown. This may not recover until
+  billing quota resets or the org is changed.
+- Request rate limited: short cooldown.
+- Concurrent stream limit: very short cooldown. Capacity may return as soon as
+  another stream closes, possibly in another process.
+- Invalid/auth key: disable for this process.
+- Network/server/transient errors: short cooldown.
+
+## Scope
+
+This hotfix targets Cloud v1 transcription and translation streams that use
+Soniox.

cloud/packages/cloud/src/services/session/soniox/SonioxKeyPool.ts

@@ -0,0 +1,251 @@
+import crypto from "crypto";
+
+export type SonioxCredentialRole = "primary" | "fallback";
+
+export interface SonioxCredential {
+  id: string;
+  apiKey: string;
+  role: SonioxCredentialRole;
+}
+
+type SonioxCredentialFailureKind =
+  | "auth"
+  | "concurrency"
+  | "quota"
+  | "rate_limit"
+  | "transient";
+
+interface SonioxCredentialState extends SonioxCredential {
+  cooldownUntil: number;
+  disabled: boolean;
+  failureKind?: SonioxCredentialFailureKind;
+  lastFailureMessage?: string;
+}
+
+export interface SonioxCredentialFailureClassification {
+  kind: SonioxCredentialFailureKind;
+  cooldownMs: number;
+  disabled?: boolean;
+}
+
+const CONCURRENCY_COOLDOWN_MS = 5_000;
+const RATE_LIMIT_COOLDOWN_MS = 60_000;
+const QUOTA_COOLDOWN_MS = 30 * 60_000;
+const TRANSIENT_COOLDOWN_MS = 10_000;
+const sharedPools = new Map<string, SonioxKeyPool>();
+
+export function parseSonioxFallbackApiKeys(value: string | undefined): string[] {
+  if (!value) return [];
+  return value
+    .split(",")
+    .map((key) => key.trim())
+    .filter(Boolean);
+}
+
+export function fingerprintSonioxKey(apiKey: string): string {
+  return crypto.createHash("sha256").update(apiKey).digest("hex").slice(0, 12);
+}
+
+export function classifySonioxCredentialFailure(error: Error): SonioxCredentialFailureClassification {
+  const message = error.message || "";
+  const lower = message.toLowerCase();
+  const code = extractSonioxErrorCode(message);
+
+  if (
+    code === 401 ||
+    lower.includes("invalid api key") ||
+    lower.includes("invalid_api_key") ||
+    lower.includes("bad api key") ||
+    lower.includes("unauthorized")
+  ) {
+    return { kind: "auth", cooldownMs: Number.POSITIVE_INFINITY, disabled: true };
+  }
+
+  if (
+    lower.includes("concurrent") ||
+    lower.includes("concurrency") ||
+    lower.includes("connection limit") ||
+    lower.includes("stream limit") ||
+    lower.includes("too many streams") ||
+    lower.includes("maximum streams") ||
+    lower.includes("max streams")
+  ) {
+    return { kind: "concurrency", cooldownMs: CONCURRENCY_COOLDOWN_MS };
+  }
+
+  if (
+    code === 429 ||
+    lower.includes("rate limit") ||
+    lower.includes("rate_limit") ||
+    lower.includes("too many requests")
+  ) {
+    return { kind: "rate_limit", cooldownMs: RATE_LIMIT_COOLDOWN_MS };
+  }
+
+  if (
+    code === 402 ||
+    /\bquota\b/.test(lower) ||
+    /\bbudget\b/.test(lower) ||
+    /\bcredit(?:s)?\b/.test(lower) ||
+    /\bbilling\b/.test(lower) ||
+    /\bspend(?:ing)?\b/.test(lower) ||
+    /\bbalance\b/.test(lower) ||
+    lower.includes("usage limit") ||
+    lower.includes("monthly limit")
+  ) {
+    return { kind: "quota", cooldownMs: QUOTA_COOLDOWN_MS };
+  }
+
+  return { kind: "transient", cooldownMs: TRANSIENT_COOLDOWN_MS };
+}
+
+export function getSharedSonioxKeyPool(primaryApiKey: string, fallbackApiKeys: string[] = []): SonioxKeyPool {
+  const poolKey = [
+    fingerprintSonioxKey(primaryApiKey.trim()),
+    ...fallbackApiKeys.map((key) => key.trim()).filter(Boolean).map(fingerprintSonioxKey),
+  ].join(":");
+
+  const existing = sharedPools.get(poolKey);
+  if (existing) return existing;
+
+  const pool = new SonioxKeyPool(primaryApiKey, fallbackApiKeys);
+  sharedPools.set(poolKey, pool);
+  return pool;
+}
+
+export function resetSharedSonioxKeyPoolsForTests(): void {
+  sharedPools.clear();
+}
+
+export class SonioxKeyPool {
+  private credentials: SonioxCredentialState[];
+  private nextFallbackIndex = 0;
+
+  constructor(primaryApiKey: string, fallbackApiKeys: string[] = []) {
+    const seen = new Set<string>();
+    const credentials: SonioxCredentialState[] = [];
+
+    const addCredential = (apiKey: string, role: SonioxCredentialRole): void => {
+      const trimmed = apiKey.trim();
+      if (!trimmed || seen.has(trimmed)) return;
+      seen.add(trimmed);
+      credentials.push({
+        id: fingerprintSonioxKey(trimmed),
+        apiKey: trimmed,
+        role,
+        cooldownUntil: 0,
+        disabled: false,
+      });
+    };
+
+    addCredential(primaryApiKey, "primary");
+    for (const key of fallbackApiKeys) {
+      addCredential(key, "fallback");
+    }
+
+    this.credentials = credentials;
+  }
+
+  get size(): number {
+    return this.credentials.length;
+  }
+
+  get hasFallbacks(): boolean {
+    return this.credentials.some((credential) => credential.role === "fallback");
+  }
+
+  selectCredential(attempted = new Set<string>(), now = Date.now()): SonioxCredential | null {
+    const primary = this.credentials.find((credential) => credential.role === "primary");
+    if (primary && !attempted.has(primary.id) && this.isAvailable(primary, now)) {
+      return this.toPublicCredential(primary);
+    }
+
+    const fallbackCredentials = this.credentials.filter((credential) => credential.role === "fallback");
+    if (fallbackCredentials.length === 0) return null;
+
+    for (let offset = 0; offset < fallbackCredentials.length; offset++) {
+      const index = (this.nextFallbackIndex + offset) % fallbackCredentials.length;
+      const credential = fallbackCredentials[index];
+      if (attempted.has(credential.id) || !this.isAvailable(credential, now)) continue;
+
+      this.nextFallbackIndex = (index + 1) % fallbackCredentials.length;
+      return this.toPublicCredential(credential);
+    }
+
+    return null;
+  }
+
+  recordSuccess(credentialId: string, now = Date.now()): void {
+    const credential = this.findCredential(credentialId);
+    if (!credential || credential.disabled) return;
+    if (credential.cooldownUntil > now) return;
+    credential.cooldownUntil = 0;
+    credential.failureKind = undefined;
+    credential.lastFailureMessage = undefined;
+  }
+
+  recordFailure(credentialId: string, error: Error, now = Date.now()): SonioxCredentialFailureClassification | null {
+    const credential = this.findCredential(credentialId);
+    if (!credential) return null;
+
+    const classification = classifySonioxCredentialFailure(error);
+    credential.failureKind = classification.kind;
+    credential.lastFailureMessage = error.message;
+
+    if (classification.disabled) {
+      credential.disabled = true;
+      credential.cooldownUntil = Number.POSITIVE_INFINITY;
+    } else {
+      credential.cooldownUntil = Math.max(
+        credential.cooldownUntil,
+        now + classification.cooldownMs,
+      );
+    }
+
+    return classification;
+  }
+
+  describeAvailability(now = Date.now()): Array<{
+    id: string;
+    role: SonioxCredentialRole;
+    available: boolean;
+    disabled: boolean;
+    cooldownRemainingMs: number;
+    failureKind?: SonioxCredentialFailureKind;
+  }> {
+    return this.credentials.map((credential) => ({
+      id: credential.id,
+      role: credential.role,
+      available: this.isAvailable(credential, now),
+      disabled: credential.disabled,
+      cooldownRemainingMs:
+        credential.cooldownUntil === Number.POSITIVE_INFINITY
+          ? Number.POSITIVE_INFINITY
+          : Math.max(0, credential.cooldownUntil - now),
+      failureKind: credential.failureKind,
+    }));
+  }
+
+  private findCredential(credentialId: string): SonioxCredentialState | undefined {
+    return this.credentials.find((credential) => credential.id === credentialId);
+  }
+
+  private isAvailable(credential: SonioxCredentialState, now: number): boolean {
+    return !credential.disabled && credential.cooldownUntil <= now;
+  }
+
+  private toPublicCredential(credential: SonioxCredentialState): SonioxCredential {
+    return {
+      id: credential.id,
+      apiKey: credential.apiKey,
+      role: credential.role,
+    };
+  }
+}
+
+function extractSonioxErrorCode(message: string): number | null {
+  const match = message.match(/Soniox error (\d+):/i);
+  if (!match) return null;
+  const parsed = Number.parseInt(match[1], 10);
+  return Number.isFinite(parsed) ? parsed : null;
+}