KIP-30: Sandbox MCP productionization (M0 done, M1-M3 proposed)

cmd/sandboxcli/main.go

@@ -58,6 +58,7 @@ func main() {
 		sandboxcli.ExposedCommand(),
 		sandboxcli.AllowHostsCommand(),
 		sandboxcli.BenchmarkCommand(),
+		sandboxcli.TLSCommand(),
 	}
 	// M9 catalog: emits the full command surface (single source for SDK stubs).
 	commands = append(commands, sandboxcli.CatalogCommand(commands))

docs/README.md

@@ -45,6 +45,7 @@ Last audited: **2026-08-24**, against the in-tree sandbox matrix, gRPC proto,
 | [KIP-14](kip-14-mtls-dynamic-cert-issuance.md) | mTLS dynamic client certs (Login RPC) | Implemented |
 | [KIP-17](kip-17-sandbox-cli-profiles-and-apikey-ttl.md) | CLI multi-profile + API key TTL | Implemented |
 | [KIP-28](kip-28-sandbox-gateway-trust-bootstrap.md) | Trust anchor bootstrap for the remote gRPC gateway | Proposed — M1 client trust chain + `--reset-trust`; M2 `/.well-known` CA distribution; M3 publicly-trusted gateway cert |
+| [KIP-30](kip-30-sandbox-mcp-productionization.md) | Sandbox MCP productionization — healthy default install, multi-node readiness, IP-only TLS | Proposed — M0 (default-skip + manual TLS) implemented and verified on a live cluster; M1–M3 open |
 
 ### Completeness / architecture reviews
 
@@ -87,6 +88,10 @@ Agent-facing repo conventions live under [`docs/agents/`](agents/). Those files
 - **KIP-28** closes the remote-trust gap left by KIP-22 (correct SAN, untrusted issuer) and
   by KIP-27 (cert-manager present, but only wired to the `:443` listener). The next free
   integer is **KIP-29**.
+- **KIP-30** was drafted while KIP-29 was still open; both numbers are now used and the
+  next free integer is **KIP-31**. It is a follow-up to KIP-8 (making the MCP Deployment
+  runnable) rather than a revision of it — KIP-8's protocol and API-key design is
+  unchanged.
 
 ## How to add a KIP
 

hack/init-sandbox-mcp.sh

@@ -6,26 +6,33 @@ kubeconfig=""
 public_cert=""
 public_key=""
 gateway_ca=""
-gateway_cert=""
-gateway_key=""
 hostname=""
 image=""
 apply=false
 
 usage() {
   cat <<'EOF'
 Usage: bash hack/init-sandbox-mcp.sh --kubeconfig FILE \
-  --gateway-ca FILE --gateway-cert FILE --gateway-key FILE \
+  [--gateway-ca FILE] \
   [--public-cert FILE --public-key FILE] \
   [--hostname MCP_PUBLIC_HOST] [--image MCP_IMAGE] [--apply]
 
-Uses existing K8E keys from sandbox-matrix/sandbox-apikeys. No OAuth server is
-needed. An existing sandbox-matrix/sandbox-e2b certificate is reused unless a
+MCP enrolls its own gateway identity (KIP-30 M2): the only material this
+script provides is the sandbox CA (defaults to
+/var/lib/k8e/server/tls/sandbox-ca.crt) and one API key from
+sandbox-matrix/sandbox-apikeys; k8e-mcp/mcp-bootstrap is the Secret that
+carries both into the Deployment. No client certificate pair is generated,
+staged or mounted.
+
+Set MCP_BOOTSTRAP_API_KEY (environment, never argv) to enroll with a specific
+key. Otherwise the first active key named in keys.json is used.
+
+An existing sandbox-matrix/sandbox-e2b certificate is reused unless a
 new public certificate pair is supplied explicitly. The existing k8e API
-Gateway exposes https://HOST/mcp. Without --apply, validates local inputs only.
-The Deployment pulls ghcr.io/xiaods/k8e-mcp:latest (published by the release
-workflow); pass --image to point it elsewhere, e.g. --image k8e-mcp:local for a
-locally built image imported into K8E containerd.
+Gateway exposes https://HOST/mcp. Without --apply, validates local inputs
+only. The Deployment pulls ghcr.io/xiaods/k8e-mcp:latest (published by the
+release workflow); pass --image to point it elsewhere, e.g. --image
+k8e-mcp:local for a locally built image imported into K8E containerd.
 
 Set MCP_PROBE_API_KEY (environment, never argv) to additionally prove the
 authenticated MCP path by calling server/discover and requiring HTTP 200.
@@ -37,7 +44,7 @@ rollout_failure_diagnostics() {
   kubectl "${kubectl_args[@]}" -n k8e-mcp get pods -o wide >&2 || true
   kubectl "${kubectl_args[@]}" -n k8e-mcp describe deployment/k8e-mcp >&2 || true
   kubectl "${kubectl_args[@]}" -n k8e-mcp logs deployment/k8e-mcp --all-containers --tail=40 >&2 || true
-  echo "check that --gateway-ca/--gateway-cert/--gateway-key are a valid, unexpired mTLS client pair for the sandbox gateway" >&2
+  echo "check that --gateway-ca is the sandbox CA and the bootstrap key exists in sandbox-matrix/sandbox-apikeys" >&2
 }
 
 while [[ $# -gt 0 ]]; do
@@ -51,16 +58,19 @@ while [[ $# -gt 0 ]]; do
     --public-cert) public_cert="$2" ;;
     --public-key) public_key="$2" ;;
     --gateway-ca) gateway_ca="$2" ;;
-    --gateway-cert) gateway_cert="$2" ;;
-    --gateway-key) gateway_key="$2" ;;
     --hostname) hostname="$2" ;;
     --image) image="$2" ;;
     *) echo "unknown option: $1" >&2; exit 2 ;;
   esac
   shift 2
 done
 
-for file in "$kubeconfig" "$gateway_ca" "$gateway_cert" "$gateway_key"; do
+# The sandbox CA on a K8E server host is a fixed, non-secret path: it exists
+# so every client can trust the gateway. Defaulting keeps the common
+# (script-on-the-server) invocation free of file flags.
+gateway_ca="${gateway_ca:-/var/lib/k8e/server/tls/sandbox-ca.crt}"
+
+for file in "$kubeconfig" "$gateway_ca"; do
   [[ -n "$file" && -r "$file" && -s "$file" ]] || { echo "required nonempty readable file: $file" >&2; exit 2; }
 done
 if [[ -n "$public_cert" || -n "$public_key" ]]; then
@@ -72,6 +82,7 @@ fi
 command -v kubectl >/dev/null || { echo "kubectl is required" >&2; exit 2; }
 command -v curl >/dev/null || { echo "curl is required" >&2; exit 2; }
 command -v openssl >/dev/null || { echo "openssl is required" >&2; exit 2; }
+command -v grep >/dev/null || { echo "grep is required" >&2; exit 2; }
 if [[ -n "$hostname" ]]; then
   valid_host=false
   if [[ ${#hostname} -le 253 && "$hostname" =~ ^([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?$ ]]; then
@@ -104,7 +115,37 @@ kubectl "${kubectl_args[@]}" create namespace k8e-mcp --dry-run=client -o yaml |
 if [[ -n "$public_cert" ]]; then
   kubectl "${kubectl_args[@]}" -n sandbox-matrix create secret tls sandbox-e2b --cert="$public_cert" --key="$public_key" --dry-run=client -o yaml | kubectl "${kubectl_args[@]}" apply --server-side --field-manager=k8e-mcp-init -f -
 fi
-kubectl "${kubectl_args[@]}" -n k8e-mcp create secret generic mcp-gateway-tls --from-file=ca.crt="$gateway_ca" --from-file=tls.crt="$gateway_cert" --from-file=tls.key="$gateway_key" --dry-run=client -o yaml | kubectl "${kubectl_args[@]}" apply --server-side --field-manager=k8e-mcp-init -f -
+
+# The enrollment key: MCP_BOOTSTRAP_API_KEY selects one explicitly, otherwise
+# the first active key named in keys.json is reused (python3 preferred, a
+# grep fallback for minimal images). argv is never used for secrets.
+bootstrap_key="${MCP_BOOTSTRAP_API_KEY:-}"
+if [[ -z "$bootstrap_key" ]]; then
+  keys_json="$(kubectl "${kubectl_args[@]}" -n sandbox-matrix get secret sandbox-apikeys -o jsonpath='{.data.keys}')"
+  [[ -n "$keys_json" ]] || { echo "sandbox-apikeys has no keys" >&2; exit 1; }
+  if command -v python3 >/dev/null; then
+    bootstrap_key="$(printf %s "$keys_json" | base64 -d | python3 -c 'import json,sys
+for entry in json.load(sys.stdin):
+    if entry.get("state") == "active":
+        print(entry["key"]); break
+')" || bootstrap_key=""
+  else
+    # Alternate key/state output lines; key precedes state in each entry.
+    while IFS= read -r key_line && IFS= read -r state_line; do
+      if [[ "$state_line" == *active* ]]; then
+        bootstrap_key="${key_line//[^0-9a-f]/}"
+        break
+      fi
+    done < <(printf '%s' "$keys_json" | base64 -d | grep -oE '"(key|state)" *: *"[^"]*"')
+  fi
+fi
+[[ -n "$bootstrap_key" ]] || { echo "no unexpired API key found in sandbox-matrix/sandbox-apikeys (set MCP_BOOTSTRAP_API_KEY to choose one)" >&2; exit 1; }
+
+# k8e-mcp/mcp-bootstrap is all the Deployment needs to enroll: the sandbox CA
+# (non-secret; distributed to every client) and one API key. mcp-serve
+# performs the Login itself and caches the resulting identity in ConfigMap
+# k8e-mcp/mcp-client-cert.
+kubectl "${kubectl_args[@]}" -n k8e-mcp create secret generic mcp-bootstrap --from-file=ca.crt="$gateway_ca" --from-literal=apikey="$bootstrap_key" --dry-run=client -o yaml | kubectl "${kubectl_args[@]}" apply --server-side --field-manager=k8e-mcp-init -f -
 # The server default-skips manifests/sandbox-mcp (pkg/server stageFiles): the
 # Deployment cannot become ready without this script's Secret, so this apply is
 # load-bearing, not redundant.

manifests/sandbox-matrix/crds.yaml

@@ -169,6 +169,13 @@ spec:
               minSize: {type: integer, minimum: 0}
               maxSize: {type: integer, minimum: 0}
               idleTTLSeconds: {type: integer, minimum: 0}
+              # KIP-30 M3.3: constrain which nodes pool pods may land on.
+              # Merged into the warm pod nodeSelector; the controller adds a
+              # required affinity on nodes annotated with the pool's
+              # RuntimeClass (sandbox.k8e.io/runtime-classes: gvisor,kata).
+              nodeSelector:
+                type: object
+                additionalProperties: {type: string}
               templateRef:
                 type: object
                 properties:

manifests/sandbox-matrix/e2b-gateway.yaml

@@ -23,10 +23,13 @@
 #     and the sandbox gRPC gateway
 #     enabled (0.0.0.0:50051)
 #
-# Both host services are bridged into the cluster via headless Service +
-# discovery.k8s.io/v1 EndpointSlice pointing at --advertise-address (KIP-21:
-# never loopback — Kubernetes does not allow loopback Endpoint addresses),
-# because they run in the k8e-server host process, not as pods.
+# Both host services are bridged into the cluster via headless Services,
+# because they run in the k8e-server host process, not as pods. The
+# sandbox-grpc-gateway EndpointSlice is owned by the leader-gated
+# sandboxmatrix controller (KIP-30 M3.1: one endpoint per server node's
+# routable addresses, reconciled from live cluster state); the e2b-server
+# slice is staged statically from --advertise-address (KIP-21: never
+# loopback — Kubernetes does not allow loopback Endpoint addresses).
 ---
 
 apiVersion: v1
@@ -47,26 +50,11 @@ spec:
   - name: grpc
     port: 50051
     protocol: TCP
----
-apiVersion: discovery.k8s.io/v1
-kind: EndpointSlice
-metadata:
-  name: sandbox-grpc-gateway
-  namespace: %{SANDBOX_NAMESPACE}%
-  labels:
-    kubernetes.io/service-name: sandbox-grpc-gateway
-    endpointslice.kubernetes.io/managed-by: k8e
-    app.kubernetes.io/managed-by: k8e
-addressType: IPv4
-endpoints:
-- addresses:
-  - "%{ADVERTISE_IP}%"
-  conditions:
-    ready: true
-ports:
-- name: grpc
-  port: 50051
-  protocol: TCP
+# The sandbox-grpc-gateway EndpointSlice is NOT staged here (KIP-30 M3.1):
+# the leader-gated sandboxmatrix controller owns it and writes one endpoint
+# per server node's routable addresses, so HA membership follows the live
+# cluster instead of a stage-time %{ADVERTISE_IP}%. The Service below is the
+# stable name both the TCPRoute and in-cluster clients dial.
 ---
 apiVersion: v1
 kind: Service
@@ -88,6 +76,9 @@ spec:
   - name: http
     port: 3676
     protocol: TCP
+# e2b-server keeps its staged EndpointSlice: it is a single host-process
+# service (embedded e2b in the k8e-server process), unlike the gRPC gateway
+# which HA-deploys on every server node.
 ---
 apiVersion: discovery.k8s.io/v1
 kind: EndpointSlice

manifests/sandbox-mcp/README.md

@@ -9,7 +9,7 @@ resource configuration or OAuth client secret is required.
 The deployment is **opt-in**: the K8E server does not stage this manifest by
 default (see `pkg/server` `stageFiles`). Every default install ran a
 `k8e-mcp` Deployment whose pods could never become ready — the
-`mcp-gateway-tls` Secret and the `k8e-mcp:local` image below do not exist
+`mcp-bootstrap` Secret and the `k8e-mcp:local` image below do not exist
 until an operator provides them — so the manifest is only applied by
 `hack/init-sandbox-mcp.sh` once its prerequisites are in place. Skipping is
 fail-closed: upgrading a server removes a previously staged copy, and a
@@ -42,9 +42,7 @@ there), and pass `--image` so the Deployment uses the local tag:
 docker build -f manifests/sandbox-mcp/Dockerfile -t k8e-mcp:local .
 # Import into the target node's containerd (k8s.io namespace):
 docker save k8e-mcp:local | docker exec -i <k8e-node-container> ctr -n k8s.io images import -
-bash hack/init-sandbox-mcp.sh --kubeconfig /path/to/k8e.yaml \
-  --gateway-ca ... --gateway-cert ... --gateway-key ... \
-  --image k8e-mcp:local --apply
+bash hack/init-sandbox-mcp.sh --kubeconfig /path/to/k8e.yaml --image k8e-mcp:local --apply
 ```
 
 The runtime image is a non-root `scratch` image containing the static CLI and
@@ -54,46 +52,63 @@ harness has no node, so MCP pods cannot schedule there). A registry deployment
 should pin an immutable digest instead: `--image
 ghcr.io/xiaods/k8e-mcp@sha256:<digest>`.
 
-### Gateway mTLS material
-
-`mcp-serve` reaches the sandbox gRPC gateway with a client certificate signed by
-the sandbox CA. Bootstrap that pair from the same API key you created above —
-the CLI performs the gateway `Login` handshake and writes `ca.crt`, `client.crt`
-and `client.key`:
+### Gateway identity (KIP-30 M2)
+
+`mcp-serve` reaches the sandbox gRPC gateway with a client certificate signed
+by the sandbox CA. Since M2 the adapter **enrolls its own identity at
+startup**: `hack/init-sandbox-mcp.sh` ships the sandbox CA
+(`/var/lib/k8e/server/tls/sandbox-ca.crt` by default; non-secret — a CA exists
+to be distributed) and one API key in Secret `k8e-mcp/mcp-bootstrap`; the
+adapter performs the gateway `Login` handshake itself and caches the resulting
+identity in ConfigMap `k8e-mcp/mcp-client-cert` (`ca.crt` / `tls.crt` /
+`tls.key`).
+
+The cache is what makes restarts, rollouts and upgrades cheap: a warm cache
+serves the identity with **zero Login calls and no API key needed**, so
+`K8E_MCP_BOOTSTRAP_API_KEY` can be unset after the first enrollment. The
+client certificate is issued with the standard 90-day TTL; a restart within
+30 days of expiry re-enrolls, **reusing the cached private key** so the
+identity (and the server's ledger entry) stays stable. First-writer-wins on
+the ConfigMap means N replicas of one Deployment share one identity. To force
+a brand-new identity, delete `k8e-mcp/mcp-client-cert`.
+
+The API key is only consumed on cold starts and renewals. Rotate it in
+`sandbox-matrix/sandbox-apikeys` and update the Secret; in-flight enrolled
+certificates remain valid until their own TTL.
 
 ```sh
-K8E_SANDBOX_CERT_DIR=/path/to/gateway-mtls \
-  k8e-sandbox-cli connect --endpoint <gateway-host>:50051 \
-  --apikey <bare-key-from-sandbox-apikey-create> --skip-skill
+bash hack/init-sandbox-mcp.sh --kubeconfig /path/to/k8e.yaml --apply
+# or choose the key explicitly (environment, never argv):
+MCP_BOOTSTRAP_API_KEY=<bare-key> \
+  bash hack/init-sandbox-mcp.sh --kubeconfig /path/to/k8e.yaml --apply
 ```
 
-Pass those three files as `--gateway-ca/--gateway-cert/--gateway-key` below. The
-files are the adapter's own backend identity: keep them out of source control and
-treat the directory as secret material. The gateway only verifies that a client
-certificate is present and unrevoked; the MCP adapter, not gRPC mTLS, is what
-enforces per-resource ownership (see `pkg/sandboxmcp/README.md`).
+The trust model is fail-closed: with no CA and no cached identity, the adapter
+**refuses to send the API key** unless `--insecure-bootstrap` /
+`K8E_MCP_INSECURE_BOOTSTRAP` is set explicitly (trust-on-first-use, pinned to
+the CA the server presents on first contact).
 
-Initialize using authorized gateway mTLS files. If the API Gateway already has
-the `sandbox-matrix/sandbox-e2b` TLS Secret, omit `--public-cert` and
-`--public-key`. Supply both options to install or deliberately update that
-shared Gateway certificate:
+Operators who pre-generate pairs keep the pre-M2 path: `--gateway-ca`,
+`--gateway-cert` and `--gateway-key` (all three, files, mounted by the
+operator) skip enrollment entirely. The two modes are mutually exclusive.
 
-```sh
-bash hack/init-sandbox-mcp.sh --kubeconfig /path/to/k8e.yaml \
-  --gateway-ca /path/to/gateway/ca.crt \
-  --gateway-cert /path/to/gateway/tls.crt --gateway-key /path/to/gateway/tls.key \
-  --public-cert /path/to/public/tls.crt --public-key /path/to/public/tls.key \
-  --hostname mcp.example.com \
-  --apply
-```
+### Tenant scoping
+
+`K8E_MCP_TENANT` scopes every session the adapter creates to one tenant
+(tenants are the isolation unit; API keys are only credentials). Empty keeps
+the pre-M2 behavior — sessions land in the server-side shared `default` bucket,
+which is single-tenant only. Run one Deployment per tenant for more, each with
+its own `mcp-bootstrap` Secret namespace copy and its own state namespace.
 
 Without `--apply`, only local input checks run. With it, the script verifies the
 existing API key Secret and the K8E API Gateway, creates the namespace, verifies
-or explicitly updates `sandbox-matrix/sandbox-e2b`, creates the gateway mTLS
-client Secret, applies the Deployment, Service, ReferenceGrant and HTTPRoute,
-then waits for the Deployment, the Gateway's `https` listener and route
-references. `--image` overrides the Deployment image (default
-`ghcr.io/xiaods/k8e-mcp:latest`) for locally built or air-gap-imported images.
+or explicitly updates `sandbox-matrix/sandbox-e2b`, picks an enrollment key
+(`MCP_BOOTSTRAP_API_KEY` or the first active key in `keys.json`), creates
+`k8e-mcp/mcp-bootstrap` (CA + key), applies the Deployment, Service,
+ReferenceGrant and HTTPRoute, then waits for the Deployment, the Gateway's
+`https` listener and route references. `--image` overrides the Deployment image
+(default `ghcr.io/xiaods/k8e-mcp:latest`) for locally built or air-gap-imported
+images.
 If the Deployment does not become ready it prints pod status,
 the deployment description and recent logs, then exits non-zero. Before
 reporting success, it makes an unauthenticated HTTPS request to `/mcp` and
@@ -112,11 +127,14 @@ certificate and DNS record. When omitted, the script uses the first address from
 contains that IP.
 
 RBAC grants ConfigMap get/create/update in `k8e-mcp`, plus get on exactly
-`sandbox-matrix/sandbox-apikeys`. The deployment uses two replicas and a disruption
+`sandbox-matrix/sandbox-apikeys`. The identity ConfigMap `mcp-client-cert` is
+covered by an explicit resourceName rule. The deployment uses two replicas and a disruption
 budget, non-root execution and a read-only filesystem. `/healthz` reports process
 health only; `/readyz` additionally verifies the ConfigMap state store and the
 sandbox gateway, so a replica that cannot serve tool calls leaves Service
-endpoints instead of accepting traffic. The HTTPRoute lives in `sandbox-matrix` so it can attach to the
+endpoints instead of accepting traffic. The startup probe is a plain TCP check —
+it passes once the listener is up, before enrollment completes; readiness is
+what fails a replica whose identity cannot be established. The HTTPRoute lives in `sandbox-matrix` so it can attach to the
 Gateway's namespace-local HTTPS listener. A narrow ReferenceGrant in `k8e-mcp`
 allows only that HTTPRoute kind and namespace to reference Service `k8e-mcp`.
 Authorization headers and MCP metadata pass through unchanged. Public DNS still