wandb · zacharyblasczyk · Aug 5, 2026 · Jul 27, 2026 · Jul 29, 2026 · Jul 29, 2026
@@ -30,7 +30,7 @@ jobs:
           helm plugin install https://github.com/origranot/helm-cascade
           helm cascade build ./charts/operator-wandb/
           helm plugin install https://github.com/helm-unittest/helm-unittest.git --version v1.1.1
-          helm unittest --file 'tests/azure_storage_auth_test.yaml' ./charts/operator-wandb/
+          helm unittest ./charts/operator-wandb/
           helm plugin install https://github.com/jlandowner/helm-chartsnap
           ./snapshots.sh run
       # currently this action always tries to install helm, so dont use it for now

@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.44.0
+version: 0.44.1
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 

@@ -231,6 +231,7 @@ The following credentials can be pulled from external Kubernetes Secrets:
 | **ClickHouse** | `global.clickhouse.*` | Each field (host, port, database, user, password) can be a string or a map with `valueFrom` |
 | **Kafka** | `global.kafka.passwordSecret` | `name`, `passwordKey` |
 | **OIDC** | `global.auth.oidc.oidcSecret` | `name`, `secretKey` |
+| **Session signing** | `global.auth.sessionKey`, `global.auth.sessionKeyPrevious` | Literal value or a map with `valueFrom` |
 | **SMTP** | `global.email.smtp.*` | Each field (host, port, user, password) can be a string or a map with `valueFrom` |
 
 ### Example: Using External Secrets with MySQL/ClickHouse/SMTP
@@ -270,6 +271,17 @@ For complete examples with secrets and additional configurations, see:
 - Values: [test-configs/operator-wandb/user-defined-secrets.yaml](../../test-configs/operator-wandb/user-defined-secrets.yaml)
 - Secrets: [test-configs/additional-resources/user-defined-secrets/](../../test-configs/additional-resources/user-defined-secrets/)
 
+### Session Key Rotation
+
+The chart generates `GORILLA_SESSION_KEY` on first install and retains it in the
+release's `gorilla-session-key` Secret. It can rotate that managed key without
+invalidating active sessions by using a three-phase workflow, or perform an
+explicit emergency hard cutover. External Secret users can force the required
+API and app rollouts with `global.auth.sessionKeyRolloutId`.
+
+See [Session Key Rotation](docs/session-key-rotation.md) for the managed
+workflow and external Secret examples.
+
 ## Chart Relationship
 
 The operator-wandb chart uses the wandb-base chart as a building block for deploying various W&B services. The wandb-base chart provides a consistent deployment pattern for different services, while allowing for service-specific configuration.

@@ -0,0 +1,133 @@
+# Session Key Rotation
+
+`operator-wandb` generates `GORILLA_SESSION_KEY` on first install and stores it
+in `<release>-gorilla-session-key`. Later reconciliations retain the stored key.
+
+The chart can rotate this managed key without immediately invalidating existing
+sessions. The routine workflow has three phases so every running pod learns the
+candidate key before any pod starts signing with it.
+
+## Managed Rotation
+
+Choose a unique rotation ID and start with `prepare`:
+
+```yaml
+global:
+  auth:
+    sessionKeyRotation:
+      id: "2026-07-session-key"
+      phase: prepare
+```
+
+`prepare` generates a candidate key, keeps the current signing key, and adds the
+candidate to `GORILLA_SESSION_PREVIOUS_KEYS`. The rotation ID and phase also
+change the API and app pod templates, which starts a rollout. Wait for every
+enabled API and app pod to finish rolling before continuing. This wait is the
+critical safety step: during the next rollout, prepared pods sign with A and
+accept B, while activated pods sign with B and accept A.
+
+Promote the candidate with the same ID:
+
+```yaml
+global:
+  auth:
+    sessionKeyRotation:
+      id: "2026-07-session-key"
+      phase: activate
+```
+
+`activate` makes the candidate the current signing key and retains the outgoing
+key in `GORILLA_SESSION_PREVIOUS_KEYS`. Reapplying the same ID and phase does not
+swap the keys again.
+
+After the longest-lived session or token has expired, remove the verification
+keys:
+
+```yaml
+global:
+  auth:
+    sessionKeyRotation:
+      id: "2026-07-session-key"
+      phase: clear
+```
+
+Wait for each rollout to complete before advancing the phase. Use a new ID for
+the next rotation. To cancel before activation, advance a prepared rotation
+directly to `clear`; the current signing key will remain unchanged.
+
+## Choose a Rotation Policy
+
+The phase sequence and wait policy determine the response:
+
+| Policy | Procedure | User impact | Use when |
+| ------ | --------- | ----------- | -------- |
+| Routine graceful | `prepare`, `activate`, wait out the longest-lived cookie or token, then `clear` | Existing sessions and tokens remain valid | Scheduled rotation with no suspected exposure |
+| Accelerated | `prepare` and wait, `activate` and wait, then `clear` immediately | Avoids cross-generation login loops, but invalidates sessions and tokens signed by the old key | A leak is plausible, but there is no evidence of active forgery |
+| Emergency hard cutover | Apply `hard-cutover` with a new rotation ID | Logs users out and may cause transient authentication failures during rollout | There is high confidence of compromise or active abuse |
+
+For an accelerated rotation, "immediately" means there is no token-lifetime
+overlap period after the activation rollout. Do not advance the phase while a
+rollout is still in progress.
+
+An emergency hard cutover generates a new managed current key and removes all
+previous verification keys in one reconciliation:
+
+```yaml
+global:
+  auth:
+    sessionKeyRotation:
+      id: "incident-2026-07-29"
+      phase: hard-cutover
+```
+
+This deliberately breaks compatibility between old and new pods. Coordinate it
+as a disruptive change and complete the API and app rollouts as quickly as is
+safe. Reapplying the same ID and phase retains the generated key; use a unique
+ID for every hard cutover. A hard cutover with a new ID can interrupt a
+prepared or activated rotation.
+
+## External Secrets
+
+`global.auth.sessionKey` and `global.auth.sessionKeyPrevious` accept literal
+values or Kubernetes `valueFrom` maps. Prefer Secret references so key material
+does not appear in rendered manifests:
+
+```yaml
+global:
+  auth:
+    sessionKey:
+      valueFrom:
+        secretKeyRef:
+          name: gorilla-session-keys
+          key: current
+    sessionKeyPrevious:
+      valueFrom:
+        secretKeyRef:
+          name: gorilla-session-keys
+          key: previous
+    sessionKeyRolloutId: "2026-07-session-key-prepare"
+```
+
+For external keys, create and update `gorilla-session-keys` separately.
+Kubernetes does not refresh environment variables in existing containers.
+However, an installed Secret-reload controller such as Stakater Reloader may
+detect the referenced Secret update and start API and app rollouts immediately.
+Monitor for that rollout before making another change.
+
+Do not rely on a reload controller being installed or enabled. After each
+Secret update, change `sessionKeyRolloutId` and apply the Helm release to
+guarantee that the pod template changes and every enabled API and app replica
+restarts. If a reload controller already completed a rollout, changing the
+marker may cause a second, safe rollout:
+
+1. **Prepare:** set `current` to A and `previous` to B, set the rollout ID to a
+   unique value ending in `-prepare`, apply the release, and wait for rollout.
+2. **Activate:** set `current` to B and `previous` to A, change the rollout ID
+   to a value ending in `-activate`, apply the release, and wait for rollout.
+3. **Clear:** set `previous` to an empty value, change the rollout ID to a value
+   ending in `-clear`, apply the release, and wait for rollout.
+
+`sessionKeyRolloutId` is only a pod-template marker; it must not contain key
+material. External key overrides cannot be combined with
+`sessionKeyRotation`, but they can and should be combined with
+`sessionKeyRolloutId`.
@@ -0,0 +1,27 @@
+{{- define "wandb.sessionKeyEnvs" -}}
+  {{- with .Values.global.auth.sessionKey }}
+- name: GORILLA_SESSION_KEY
+    {{- if kindIs "map" . }}
+{{- toYaml . | nindent 2 }}
+    {{- else }}
+  value: {{ . | toString | quote }}
+    {{- end }}
+  {{- end }}
+  {{- with .Values.global.auth.sessionKeyPrevious }}
+- name: GORILLA_SESSION_PREVIOUS_KEYS
+    {{- if kindIs "map" . }}
+{{- toYaml . | nindent 2 }}
+    {{- else }}
+  value: {{ . | toString | quote }}
+    {{- end }}
+  {{- end }}
+  {{- $rotation := .Values.global.auth.sessionKeyRotation -}}
+  {{- if and $rotation.id $rotation.phase }}
+- name: WANDB_SESSION_KEY_ROTATION
+  value: {{ printf "%s-%s" $rotation.id $rotation.phase | quote }}
+  {{- end }}
+  {{- with .Values.global.auth.sessionKeyRolloutId }}
+- name: WANDB_SESSION_KEY_ROLLOUT
+  value: {{ . | toString | quote }}
+  {{- end }}
+{{- end -}}
@@ -1,18 +1,99 @@
+{{- $rotation := .Values.global.auth.sessionKeyRotation -}}
+{{- $rotationID := $rotation.id | default "" -}}
+{{- $rotationPhase := $rotation.phase | default "" -}}
+{{- if ne (empty $rotationID) (empty $rotationPhase) -}}
+  {{- fail "global.auth.sessionKeyRotation.id and phase must be set together" -}}
+{{- end -}}
+{{- if and $rotationID (not (has $rotationPhase (list "prepare" "activate" "clear" "hard-cutover"))) -}}
+  {{- fail "global.auth.sessionKeyRotation.phase must be one of prepare, activate, clear, or hard-cutover" -}}
+{{- end -}}
+{{- if and $rotationID (or (not (empty .Values.global.auth.sessionKey)) (not (empty .Values.global.auth.sessionKeyPrevious))) -}}
+  {{- fail "global.auth.sessionKeyRotation cannot be combined with sessionKey or sessionKeyPrevious" -}}
+{{- end -}}
+
+{{- $secretName := printf "%s-gorilla-session-key" .Release.Name -}}
+{{- $secretObj := (lookup "v1" "Secret" .Release.Namespace $secretName) | default dict -}}
+{{- $secretData := (get $secretObj "data") | default dict -}}
+{{- $secretMetadata := (get $secretObj "metadata") | default dict -}}
+{{- $secretAnnotations := (get $secretMetadata "annotations") | default dict -}}
+{{- $storedRotationID := (get $secretAnnotations "wandb.ai/session-key-rotation-id") | default "" -}}
+{{- $storedRotationPhase := (get $secretAnnotations "wandb.ai/session-key-rotation-phase") | default "" -}}
+{{- $currentKey := (get $secretData "GORILLA_SESSION_KEY") | default (randAlphaNum 32 | b64enc) -}}
+{{- $previousKeys := (get $secretData "GORILLA_SESSION_PREVIOUS_KEYS") | default "" -}}
+
+{{- if $rotationID -}}
+  {{- if eq $rotationPhase "hard-cutover" -}}
+    {{- if not (and (eq $storedRotationID $rotationID) (eq $storedRotationPhase "hard-cutover")) -}}
+      {{- if eq $storedRotationID $rotationID -}}
+        {{- fail "global.auth.sessionKeyRotation.id must be unique for each rotation" -}}
+      {{- end -}}
+      {{- $currentKey = randAlphaNum 32 | b64enc -}}
+    {{- end -}}
+    {{- $previousKeys = "" -}}
+  {{- else if eq $rotationPhase "prepare" -}}
+    {{- if and (eq $storedRotationID $rotationID) (eq $storedRotationPhase "prepare") -}}
+      {{- /* Reuse the candidate generated by the first prepare reconciliation. */ -}}
+    {{- else -}}
+      {{- if and $storedRotationPhase (not (has $storedRotationPhase (list "clear" "hard-cutover"))) -}}
+        {{- fail "a session key rotation is already in progress; clear it before starting another rotation" -}}
+      {{- end -}}
+      {{- if and (eq $storedRotationID $rotationID) (has $storedRotationPhase (list "clear" "hard-cutover")) -}}
+        {{- fail "global.auth.sessionKeyRotation.id must be unique for each rotation" -}}
+      {{- end -}}
+      {{- $candidateKey := randAlphaNum 32 -}}
+      {{- $previousKeyList := $candidateKey -}}
+      {{- if $previousKeys -}}
+        {{- $previousKeyList = printf "%s,%s" $candidateKey ($previousKeys | b64dec) -}}
+      {{- end -}}
+      {{- $previousKeys = $previousKeyList | b64enc -}}
+    {{- end -}}
+  {{- else if eq $rotationPhase "activate" -}}
+    {{- if or (ne $storedRotationID $rotationID) (not (has $storedRotationPhase (list "prepare" "activate"))) -}}
+      {{- fail "global.auth.sessionKeyRotation.phase=activate requires a prepared rotation with the same id" -}}
+    {{- end -}}
+    {{- if eq $storedRotationPhase "prepare" -}}
+      {{- if empty $previousKeys -}}
+        {{- fail "prepared session key rotation has no candidate key" -}}
+      {{- end -}}
+      {{- $verificationKeys := splitList "," ($previousKeys | b64dec) -}}
+      {{- if empty (first $verificationKeys) -}}
+        {{- fail "prepared session key rotation has no candidate key" -}}
+      {{- end -}}
+      {{- $candidateKey := first $verificationKeys -}}
+      {{- $outgoingKeys := list ($currentKey | b64dec) -}}
+      {{- if gt (len $verificationKeys) 1 -}}
+        {{- $outgoingKeys = concat $outgoingKeys (rest $verificationKeys) -}}
+      {{- end -}}
+      {{- $currentKey = $candidateKey | b64enc -}}
+      {{- $previousKeys = join "," $outgoingKeys | b64enc -}}
+    {{- end -}}
+  {{- else if eq $rotationPhase "clear" -}}
+    {{- if or (ne $storedRotationID $rotationID) (not (has $storedRotationPhase (list "prepare" "activate" "clear"))) -}}
+      {{- fail "global.auth.sessionKeyRotation.phase=clear requires a prepared or activated rotation with the same id" -}}
+    {{- end -}}
+    {{- $previousKeys = "" -}}
+  {{- end -}}
+{{- end -}}
+
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Release.Name }}-gorilla-session-key
+  name: {{ $secretName }}
   annotations:
     "helm.sh/resource-policy": "keep"
+{{- if $rotationID }}
+    "wandb.ai/session-key-rotation-id": {{ $rotationID | quote }}
+    "wandb.ai/session-key-rotation-phase": {{ $rotationPhase | quote }}
+{{- end }}
   labels:
     {{- include "wandb.commonLabels" . | nindent 4 }}
 type: Opaque
 data:
   # Retrieve the secret data using lookup function and when not exists, return an empty dictionary / map as result
-{{- $secretObj := (lookup "v1" "Secret" .Release.Namespace (printf "%s-gorilla-session-key" .Release.Name)) | default dict }}
-{{- $secretData := (get $secretObj "data") | default dict }}
   # Set $gorillaSessionKey to existing secret data or generate a random one when not exists
-{{- $gorillaSessionKey := (get $secretData "GORILLA_SESSION_KEY") | default (randAlphaNum 32 | b64enc) }}
-  GORILLA_SESSION_KEY: {{ $gorillaSessionKey | quote }}
+  GORILLA_SESSION_KEY: {{ $currentKey | quote }}
+{{- if $previousKeys }}
+  GORILLA_SESSION_PREVIOUS_KEYS: {{ $previousKeys | quote }}
+{{- end }}
 {{- $jwkUrl := ( .Values.global.auth.jwkUrl | default (printf "http://%s-app:8083/api/jwks.json" .Release.Name)) | b64enc  }}
   GORILLA_AUTH_JWK_URL: {{ $jwkUrl | quote }}