From 7bc33ab251bcabebcfaa97bef9bb40aaa19944da Mon Sep 17 00:00:00 2001 From: zblocker64 Date: Thu, 9 Jul 2026 10:44:32 -0500 Subject: [PATCH 1/4] feat: gpu interconnect docs --- .../akash-sdl/syntax-reference/index.mdx | 10 + .../core-concepts/gpu-deployments/index.md | 399 +++++++++++++++ .../gpu-operator-migration/index.md | 211 ++++++++ .../kubespray/gpu-support/index.md | 482 ++++++++++++------ 4 files changed, 948 insertions(+), 154 deletions(-) create mode 100644 src/content/Docs/providers/operations/gpu-operator-migration/index.md diff --git a/src/content/Docs/developers/deployment/akash-sdl/syntax-reference/index.mdx b/src/content/Docs/developers/deployment/akash-sdl/syntax-reference/index.mdx index fc748fc65..f1137bf1d 100644 --- a/src/content/Docs/developers/deployment/akash-sdl/syntax-reference/index.mdx +++ b/src/content/Docs/developers/deployment/akash-sdl/syntax-reference/index.mdx @@ -347,12 +347,20 @@ gpu: - model: rtx4090 ram: 24Gi # Optional: filter by VRAM interface: pcie # Optional: pcie or sxm + interconnect: [] # Optional: opt into GPU interconnect (RDMA) ``` **Limits:** Max: 24 GPUs per service **Common models:** `a100`, `a40`, `rtx4090`, `rtx3090`, `t4`, `v100`, `h100` +**`interconnect`** — **Optional.** Opts the service into GPU interconnect (RDMA over InfiniBand or RoCE) for high-bandwidth multi-node GPU communication (NCCL). Two forms: + +- `interconnect: []` — implicit group. All services (and replicas) that use `[]` under the same placement form one peer group and are scheduled on distinct nodes. +- `interconnect: { group: }` — explicit named group, for running multiple independent groups in one deployment. The name `auto` is reserved. + +Any service using `interconnect` requires the placement to request `capabilities/gpu-interconnect: "true"` (see [placement](#placement-section)). The provider chooses the fabric and auto-injects the NCCL environment. See [GPU Interconnect (multi-node RDMA)](/docs/learn/core-concepts/gpu-deployments/#gpu-interconnect-multi-node-rdma) for full examples. +
@@ -366,6 +374,8 @@ placement: akash: attributes: # Optional filters region: us-west + capabilities/gpu-interconnect: "true" # Required if any service uses gpu.attributes.interconnect + capabilities/gpu-interconnect/fabric/infiniband: "true" # Optional: require InfiniBand specifically (or .../fabric/roce) pricing: # Required: max bid per service web: denom: uact diff --git a/src/content/Docs/learn/core-concepts/gpu-deployments/index.md b/src/content/Docs/learn/core-concepts/gpu-deployments/index.md index 7b33e89d3..db712ed37 100644 --- a/src/content/Docs/learn/core-concepts/gpu-deployments/index.md +++ b/src/content/Docs/learn/core-concepts/gpu-deployments/index.md @@ -801,10 +801,409 @@ deployment: --- +## GPU Interconnect (Multi-Node RDMA) + +Large models that don't fit on a single node need **multiple GPU nodes working as one job**, exchanging gradients or activations over a high-speed fabric. Akash exposes this as **GPU interconnect** — RDMA over InfiniBand or RoCE — through one SDL attribute: `gpu.attributes.interconnect`. + +When you opt in, the provider: + +- Schedules the peer services/replicas on **distinct nodes** (so each gets its own GPUs and RDMA devices). +- Attaches one RDMA device handle per GPU. These handles are shared from the node's physical HCA(s) — the device plugin exposes a pool (up to 63 per node), so a service with `gpu.units: 8` requests 8 handles, not 8 physical cards. +- **Auto-injects the NCCL environment** (`NCCL_IB_DISABLE=0`, `NCCL_IB_HCA`, and `NCCL_IB_GID_INDEX=3` on RoCE). You do **not** set these in your SDL — the provider fills them from the node's discovered hardware. + +You choose *whether* to use interconnect and *how peers are grouped*; the provider chooses the fabric (InfiniBand vs RoCE) unless you pin it. + +### The two forms + +```yaml +# Form A — implicit group. Every service/replica that writes `[]` under the +# same placement joins one shared group (named "auto") and is spread across +# distinct nodes. This is what most multi-node jobs want. +gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interconnect: [] + +# Form B — explicit named group. Use when you need several independent +# groups in one deployment (e.g. two separate training pairs). The name +# "auto" is reserved and cannot be used here. +gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interconnect: + group: pair0 +``` + +### Rules + +- Any service using `interconnect` **requires** the placement to request `capabilities/gpu-interconnect: "true"`. Without it the deployment is rejected. +- `gpu.units` must be greater than `0` — interconnect needs a GPU to attach a device to. It works with **any** count `≥ 1`; you are **not** required to request a full 8-GPU node. The 8-GPU examples below are just the common full-node training shape — see [Example 5](#example-5--small-footprint-single-gpu-services-in-groups) for a 1-GPU-per-service layout. +- Within one placement, don't mix the implicit (`[]`) and explicit (`{ group: ... }`) forms — pick one. +- `auto` is reserved for the implicit form; you can't write `group: auto`. + +### Example 1 — Minimal two-node job (implicit group) + +Two services on two nodes, both opting into the shared `auto` group. The worker reaches the head over Akash service DNS (`head`). + +```yaml +version: "2.0" + +services: + head: + image: my-registry/trainer:latest + env: + - RANK=0 + - WORLD_SIZE=2 + - MASTER_ADDR=head + - MASTER_PORT=29500 + expose: + - port: 29500 + to: + - service: worker + worker: + image: my-registry/trainer:latest + env: + - RANK=1 + - WORLD_SIZE=2 + - MASTER_ADDR=head + - MASTER_PORT=29500 + +profiles: + compute: + trainer: + resources: + cpu: + units: 32 + memory: + size: 128Gi + storage: + - size: 100Gi + gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interconnect: [] # implicit "auto" group + placement: + dcloud: + attributes: + capabilities/gpu-interconnect: "true" # required for interconnect + pricing: + trainer: + denom: uact + amount: 1000000 + +deployment: + head: + dcloud: + profile: trainer + count: 1 + worker: + dcloud: + profile: trainer + count: 1 +``` + +### Example 2 — Replicas as a group (single service, `count > 1`) + +A single service with `count: 2` and `interconnect: []`. Both replicas share the `auto` group, so the provider places them on distinct nodes. Use this when your launcher (e.g. `torchrun` with a rendezvous backend) handles rank assignment across identical replicas. + +```yaml +version: "2.0" + +services: + train: + image: my-registry/trainer:latest + env: + - WORLD_SIZE=2 + +profiles: + compute: + train: + resources: + cpu: + units: 32 + memory: + size: 128Gi + storage: + - size: 100Gi + gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interconnect: [] + placement: + dcloud: + attributes: + capabilities/gpu-interconnect: "true" + pricing: + train: + denom: uact + amount: 1000000 + +deployment: + train: + dcloud: + profile: train + count: 2 # two replicas → two distinct nodes, one "auto" group +``` + +### Example 3 — Explicit named group + +Functionally the same two-node pair as Example 1, but with an explicit group label. Prefer this when a deployment contains **more than one** interconnect group. + +```yaml + gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interconnect: + group: pair0 +``` + +### Example 4 — Multiple independent groups in one deployment + +Two separate GPU pairs (`pair0` and `pair1`) in the same placement. Peers within a group are spread across distinct nodes; the two groups are independent and may share nodes with each other. + +```yaml +version: "2.0" + +services: + a-head: + image: my-registry/trainer:latest + a-worker: + image: my-registry/trainer:latest + b-head: + image: my-registry/trainer:latest + b-worker: + image: my-registry/trainer:latest + +profiles: + compute: + group-a: + resources: + cpu: { units: 32 } + memory: { size: 128Gi } + storage: + - size: 100Gi + gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interconnect: + group: pair0 + group-b: + resources: + cpu: { units: 32 } + memory: { size: 128Gi } + storage: + - size: 100Gi + gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interconnect: + group: pair1 + placement: + dcloud: + attributes: + capabilities/gpu-interconnect: "true" + pricing: + group-a: { denom: uact, amount: 1000000 } + group-b: { denom: uact, amount: 1000000 } + +deployment: + a-head: { dcloud: { profile: group-a, count: 1 } } + a-worker: { dcloud: { profile: group-a, count: 1 } } + b-head: { dcloud: { profile: group-b, count: 1 } } + b-worker: { dcloud: { profile: group-b, count: 1 } } +``` + +### Example 5 — Small footprint: single-GPU services in groups + +Interconnect scales **down** as well as up. Here four single-GPU services form two independent pairs — `pair0` (`a-0`, `a-1`) and `pair1` (`b-0`, `b-1`) — for **4 GPUs total, one per service**. Each pair is spread across distinct nodes; the two pairs are independent. No 8-GPU node required. + +```yaml +version: "2.0" + +services: + a-0: + image: my-registry/trainer:latest + a-1: + image: my-registry/trainer:latest + b-0: + image: my-registry/trainer:latest + b-1: + image: my-registry/trainer:latest + +profiles: + compute: + single-gpu-a: + resources: + cpu: { units: 8 } + memory: { size: 32Gi } + storage: + - size: 50Gi + gpu: + units: 1 # one GPU per service instance + attributes: + vendor: + nvidia: + - model: a100 + interconnect: + group: pair0 + single-gpu-b: + resources: + cpu: { units: 8 } + memory: { size: 32Gi } + storage: + - size: 50Gi + gpu: + units: 1 + attributes: + vendor: + nvidia: + - model: a100 + interconnect: + group: pair1 + placement: + dcloud: + attributes: + capabilities/gpu-interconnect: "true" + pricing: + single-gpu-a: { denom: uact, amount: 200000 } + single-gpu-b: { denom: uact, amount: 200000 } + +deployment: + a-0: { dcloud: { profile: single-gpu-a, count: 1 } } + a-1: { dcloud: { profile: single-gpu-a, count: 1 } } + b-0: { dcloud: { profile: single-gpu-b, count: 1 } } + b-1: { dcloud: { profile: single-gpu-b, count: 1 } } +``` + +### Example 6 — Require a specific fabric (InfiniBand) + +By default the provider serves whichever fabric it has. If your workload specifically needs InfiniBand (or RoCE), pin it with an extra placement attribute — only providers advertising that fabric will bid. + +```yaml + placement: + dcloud: + attributes: + capabilities/gpu-interconnect: "true" + capabilities/gpu-interconnect/fabric/infiniband: "true" # or .../fabric/roce + pricing: + trainer: + denom: uact + amount: 1000000 +``` + +### Example 7 — Full multi-node LLM training (2 × 8× H100) + +A complete, deployable two-node job pinned to InfiniBand. Note there are **no `NCCL_IB_*` variables** — the provider injects them. + +```yaml +version: "2.0" + +services: + head: + image: my-registry/megatron:latest + args: + - torchrun + - --nnodes=2 + - --node_rank=0 + - --master_addr=head + - --master_port=29500 + - train.py + env: + - NCCL_DEBUG=INFO # app-level logging only; IB env is provider-injected + expose: + - port: 29500 + to: + - service: worker + worker: + image: my-registry/megatron:latest + args: + - torchrun + - --nnodes=2 + - --node_rank=1 + - --master_addr=head + - --master_port=29500 + - train.py + +profiles: + compute: + node: + resources: + cpu: + units: 64 + memory: + size: 512Gi + storage: + - size: 500Gi + attributes: + persistent: true + class: beta3 + gpu: + units: 8 + attributes: + vendor: + nvidia: + - model: h100 + interface: sxm + interconnect: [] + placement: + dcloud: + attributes: + capabilities/gpu-interconnect: "true" + capabilities/gpu-interconnect/fabric/infiniband: "true" + pricing: + node: + denom: uact + amount: 5000000 + +deployment: + head: + dcloud: + profile: node + count: 1 + worker: + dcloud: + profile: node + count: 1 +``` + +### What you still handle in your container + +The provider wires the **fabric** (RDMA devices + NCCL IB env). Your image still handles the **application-level** distributed setup: + +- Rank / world-size / master-address wiring (`RANK`, `WORLD_SIZE`, `MASTER_ADDR`, `MASTER_PORT`, or a `torchrun` rendezvous). +- Reaching peers by Akash **service name** (`head`, `worker`, …) via an `expose … to: - service: ` rule. +- Enough shared memory for your framework (set `/dev/shm` sizing in your container/launcher as usual). + +> **Verifying a provider supports interconnect:** it advertises `capabilities/gpu-interconnect` (and a `capabilities/gpu-interconnect/fabric/...` value) in its on-chain attributes. Providers set this up following [GPU & InfiniBand Support](/docs/providers/setup-and-installation/kubespray/gpu-support#part-2--infiniband--rdma-optional). + +--- + ## Related Topics - [Persistent Storage](/docs/learn/core-concepts/persistent-storage) - Storing models - [Providers & Leases](/docs/learn/core-concepts/providers-leases) - Choosing GPU providers +- [SDL syntax reference — GPU](/docs/developers/deployment/akash-sdl/syntax-reference/#gpu-section) - The `interconnect` attribute --- diff --git a/src/content/Docs/providers/operations/gpu-operator-migration/index.md b/src/content/Docs/providers/operations/gpu-operator-migration/index.md new file mode 100644 index 000000000..42d99863e --- /dev/null +++ b/src/content/Docs/providers/operations/gpu-operator-migration/index.md @@ -0,0 +1,211 @@ +--- +categories: ["Providers", "Operations"] +tags: ["GPU", "NVIDIA GPU Operator", "Migration", "Drivers", "Kubernetes", "Maintenance"] +weight: 5 +title: "GPU Operator migration" +linkTitle: "GPU Operator migration" +description: "Migrate a provider from host-installed NVIDIA drivers and a standalone device plugin to the containerized NVIDIA GPU Operator" +--- + +**Migrate an existing GPU provider from manually-installed host drivers (host `nvidia-driver-*`, NVIDIA Container Toolkit, and a standalone `nvidia-device-plugin` Helm release) to the [NVIDIA GPU Operator](/docs/providers/setup-and-installation/kubespray/gpu-support),** which manages the driver, container toolkit, device plugin, and validation in containers. + +Migrate if you want a single, versioned source of truth for the GPU stack, simpler driver upgrades, or the baseline required for the [InfiniBand / Network Operator path](/docs/providers/setup-and-installation/kubespray/gpu-support#part-2--infiniband--rdma-optional). For a **new** cluster, follow [GPU & InfiniBand Support](/docs/providers/setup-and-installation/kubespray/gpu-support) directly instead. + +> **This is disruptive.** Making the GPU Operator manage the driver requires **removing the host driver**, which means unloading the kernel module. Any GPU workload on a node stops while that node is drained and the driver reloads. Do this **one node at a time** (cordon → drain → migrate → uncordon) during a maintenance window, and expect running GPU leases on each node to be interrupted. + +> **The advertised resource is unchanged.** Both the standalone device plugin and the GPU Operator expose `nvidia.com/gpu`. Your `provider.yaml` GPU attributes do **not** change, so no re-bid or attribute edit is needed after migration. + +**Time:** ~20–30 minutes per node, plus reboot and operator rollout. + +--- + +## Prerequisites + +- Cluster admin (`kubectl`) and Helm 3 on the control plane. +- A maintenance window, or enough spare capacity to drain GPU nodes one at a time. +- Nodes were set up with the **manual** GPU workflow: a host NVIDIA driver, the NVIDIA Container Toolkit, and a standalone `nvidia-device-plugin` Helm release. + +### Confirm your current setup + +```bash +# Host driver present (this is what we will remove) +ssh nvidia-smi +ssh "dpkg -l | grep -E 'nvidia-driver|nvidia-fabricmanager|nvidia-container-toolkit'" + +# Standalone device plugin (installed manually, e.g. release "nvdp") +helm list -A | grep -i nvidia-device-plugin +kubectl -n nvidia-device-plugin get pods + +# Current GPU capacity (record this — it must match after migration) +kubectl get nodes -o custom-columns=\ +NAME:.metadata.name,\ +GPUs:.status.allocatable.nvidia\\.com/gpu +``` + +If `nvidia-smi` on the host returns nothing, the driver is **not** host-installed and you may already be on the Operator — this migration does not apply. + +--- + +## Migration overview + +``` +1. Record current state (driver/plugin versions, GPU capacity) +2. Remove the standalone nvidia-device-plugin Helm release (cluster-wide, once) +3. Per node: cordon → drain → purge host driver + toolkit → reboot +4. Install the GPU Operator (cluster-wide, once) +5. Uncordon and verify nvidia.com/gpu returns +6. Verify the provider still serves GPU leases +``` + +The standalone device plugin is removed **before** the Operator so two plugins never both try to register `nvidia.com/gpu`. + +--- + +## Step 1 — Record current state + +Note these so you can confirm parity afterward (and roll back if needed): + +- Host driver version (`nvidia-smi --query-gpu=driver_version --format=csv,noheader`) +- Standalone device plugin Helm release name, namespace, and chart version (`helm list -A`) +- GPU allocatable per node (from the prerequisites check) +- Whether nodes are SXM (need Fabric Manager) or PCIe — see the [Fabric Manager decision](/docs/providers/setup-and-installation/kubespray/gpu-support#fabric-manager-decision) + +Keep your old device-plugin values and any `RuntimeClass`/driver install notes until the migration is verified. + +## Step 2 — Remove the standalone device plugin + +Uninstall the manually-deployed plugin cluster-wide (adjust release name/namespace to your install): + +```bash +helm uninstall nvdp --namespace nvidia-device-plugin +kubectl -n nvidia-device-plugin get pods # confirm terminated +``` + +`nvidia.com/gpu` will drop to `0`/absent on the nodes until the Operator's device plugin takes over — expected, and the reason for the maintenance window. + +> Leave the existing `nvidia` `RuntimeClass` in place. The GPU Operator is compatible with it, and removing it can break scheduling for pods that request `runtimeClassName: nvidia`. + +## Step 3 — Clean each node + +Do this **one node at a time**. Repeat Step 3 fully for a node before moving to the next. + +**Cordon and drain:** + +```bash +kubectl cordon +kubectl drain --ignore-daemonsets --delete-emptydir-data --force +``` + +**Purge the host driver and toolkit** (on the node) — the inverse of the manual install: + +```bash +# GPU driver + Fabric Manager (server-driver installs used nvidia-fabricmanager) +sudo systemctl stop nvidia-fabricmanager 2>/dev/null || true +sudo apt purge --autoremove -y 'nvidia-driver-*' 'nvidia-fabricmanager-*' 'cuda-drivers*' 'nvidia-dkms-*' + +# NVIDIA Container Toolkit / runtime +sudo apt purge --autoremove -y nvidia-container-toolkit nvidia-container-runtime + +# Confirm the kernel module is gone (may require the reboot below) +lsmod | grep nvidia || echo "nvidia module unloaded" +``` + +**Reboot** to guarantee the kernel module is unloaded: + +```bash +sudo reboot +``` + +After reboot, `nvidia-smi` on the host should report **"command not found"** or no driver — that clean state is what lets the Operator manage the driver. + +> **containerd runtime:** the GPU Operator's toolkit reconfigures containerd for the `nvidia` runtime. If you previously added an `nvidia` runtime via [Kubespray Setup – Step 7](/docs/providers/setup-and-installation/kubespray/kubernetes-setup#step-7---configure-gpu-support-optional) pointing at the host binary `/usr/bin/nvidia-container-runtime`, you may remove that entry for cleanliness once the Operator is running, but it is not required — the Operator's configuration takes precedence on managed nodes. + +## Step 4 — Install the GPU Operator + +Once your GPU nodes are clean, install the Operator cluster-wide. Use the same values as a fresh setup — see [GPU & InfiniBand Support → Step 2](/docs/providers/setup-and-installation/kubespray/gpu-support#step-2--install-the-gpu-operator) for the full file and the Fabric Manager guidance: + +```bash +helm repo add nvidia https://helm.ngc.nvidia.com/nvidia +helm repo update + +helm upgrade -i gpu-operator nvidia/gpu-operator \ + --namespace gpu-operator \ + --create-namespace \ + -f gpu-operator-values.yaml +``` + +Key values for migration: + +```yaml +driver: + enabled: true # Operator installs the driver in a container + rdma: + enabled: false # Leave false unless migrating to InfiniBand too +fabricManager: + enabled: false # Set true for SXM GPUs +``` + +Watch the rollout until every pod is `Running` and `nvidia-operator-validator` completes: + +```bash +kubectl -n gpu-operator get pods -w +``` + +The Operator's driver daemonset only schedules on nodes it can manage — cleaned, uncordoned nodes. Uncordon each node after its driver pod is ready (Step 5). + +## Step 5 — Uncordon and verify + +```bash +kubectl uncordon +``` + +Confirm GPU capacity returns and matches what you recorded in Step 1: + +```bash +kubectl get nodes -o custom-columns=\ +NAME:.metadata.name,\ +GPUs:.status.allocatable.nvidia\\.com/gpu + +kubectl run gpu-test --rm -it --restart=Never \ + --image=nvidia/cuda:12.4.0-base-ubuntu22.04 \ + --limits=nvidia.com/gpu=1 \ + -- nvidia-smi +``` + +## Step 6 — Verify the provider + +Because the resource name (`nvidia.com/gpu`) is unchanged, the provider needs no attribute edits. Confirm it still bids and serves: + +```bash +# Provider pod healthy +kubectl -n akash-services get pods | grep akash-provider + +# GPU inventory reported (should match allocatable) +kubectl -n akash-services logs akash-provider-0 | grep -i gpu | tail +``` + +Deploy a small GPU workload through Akash as a final end-to-end check. + +--- + +## Rollback + +If the Operator's driver fails to come up on a node and you need GPUs back quickly: + +```bash +# Remove the Operator +helm uninstall gpu-operator --namespace gpu-operator + +# Reinstall the host driver + toolkit and the standalone device plugin +# using your Step 1 notes / previous values, then reboot the node. +``` + +Keep nodes cordoned until either the Operator driver is `Running` or the host driver is reinstalled — an uncordoned node with no working driver will accept GPU pods it cannot run. + +--- + +## Next steps + +- Adding a high-speed fabric for multi-node GPU workloads? Continue to [InfiniBand / RDMA](/docs/providers/setup-and-installation/kubespray/gpu-support#part-2--infiniband--rdma-optional). +- [Updates & maintenance](/docs/providers/operations/updates-maintenance) +- [Monitoring](/docs/providers/operations/monitoring) diff --git a/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md b/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md index 4d1699c31..a5ab79ca1 100644 --- a/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md +++ b/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md @@ -1,282 +1,456 @@ --- categories: ["Providers"] -tags: ["GPU", "Advanced Features", "Configuration"] +tags: ["GPU", "InfiniBand", "RDMA", "NVIDIA Operators", "Configuration"] weight: 2 -title: "GPU Support" +title: "GPU & InfiniBand Support" linkTitle: "GPU Support" -description: "Enable NVIDIA GPU resources on your Akash provider" +description: "Enable NVIDIA GPUs — and optional InfiniBand RDMA — on your Akash provider using the NVIDIA GPU Operator and Network Operator" --- -This guide shows how to enable NVIDIA GPU support on your Akash provider after Kubernetes is deployed. +This guide enables NVIDIA GPU support on your Akash provider using the **NVIDIA GPU Operator**, which manages the driver, container toolkit, device plugin, and validation entirely in containers — no driver installs on the host. + +It also covers an **optional** InfiniBand / RDMA section ([Part 2](#part-2--infiniband--rdma-optional)) for providers whose GPU nodes have Mellanox/NVIDIA ConnectX HCAs and need a high-speed fabric for multi-node GPU workloads (NCCL). InfiniBand is **not** required to run a GPU provider — skip Part 2 if your nodes have no IB hardware. > **Don't have GPUs?** Skip to [Persistent Storage (Rook-Ceph)](/docs/providers/setup-and-installation/kubespray/persistent-storage) or [Provider Installation](/docs/providers/setup-and-installation/kubespray/provider-installation). -> **Prerequisites:** You must have already configured the NVIDIA runtime in Kubespray **before** deploying your cluster. See [Kubernetes Setup - Step 7](/docs/providers/setup-and-installation/kubespray/kubernetes-setup#step-7---configure-gpu-support-optional). +**Time:** 30–45 minutes for GPU (add 30–60 minutes for the optional InfiniBand section, including the MOFED kernel-module compile). + +--- + +## Prerequisites + +- Kubernetes cluster deployed via [Kubespray](/docs/providers/setup-and-installation/kubespray/kubernetes-setup) (kubeadm + Calico CNI + containerd) +- **Clean GPU nodes** — no pre-installed NVIDIA drivers, CUDA, or MOFED/OFED on the host. The operators manage all drivers in containers, and a host driver will conflict. -**Time:** 30-45 minutes +For the optional InfiniBand section, additionally: + +- Mellanox/NVIDIA ConnectX InfiniBand HCAs in the GPU nodes +- InfiniBand fabric cabled and managed (a switch running a Subnet Manager, or OpenSM on a node) +- All IB ports showing `State: Active` (`ibstat` on the host) + +> **Note:** The GPU Operator replaces the older manual host-driver workflow. The hard requirement is that **no NVIDIA driver, CUDA, or MOFED/OFED is installed on the host** — remove any before starting. The GPU Operator manages the container toolkit and containerd runtime for you, so the optional NVIDIA-runtime step in [Kubernetes Setup – Step 7](/docs/providers/setup-and-installation/kubespray/kubernetes-setup#step-7---configure-gpu-support-optional) is not required for this path (it is harmless if already applied). --- -## STEP 1 - Install NVIDIA Drivers +# Part 1 — GPU Operator -Run these commands on **each GPU node**: +Every GPU provider uses this path. -### Update System +## Step 1 — Verify Clean Nodes + +Run on **each GPU node**: ```bash -apt update -DEBIAN_FRONTEND=noninteractive apt -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -apt autoremove +# Confirm no host NVIDIA driver is installed +dpkg -l | grep -E "nvidia|cuda" | grep -v lib +lsmod | grep nvidia +which nvidia-smi +# All should return empty. If a host driver is present: +# sudo apt purge --autoremove 'nvidia-*' 'cuda-drivers*' -y && sudo reboot + +# Identify GPU hardware +lspci | grep -i nvidia ``` -Reboot the node after this step. +### Fabric Manager Decision -### Add NVIDIA Repository +SXM (HGX/DGX) boards use NVLink/NVSwitch and require the Fabric Manager; PCIe cards do not. + +| GPU form factor | Fabric Manager | +| --- | --- | +| Any SXM (HGX/DGX) — A100, H100, H200, B200, B300 | Required (`fabricManager.enabled: true`) | +| Any PCIe — A100-PCIe, L40S, A6000, RTX, etc. | Not needed (`fabricManager.enabled: false`) | + +Check the form factor: ```bash -# Create keyrings directory if it doesn't exist -mkdir -p /etc/apt/keyrings +lspci -v | grep -A5 -i nvidia | grep -i "subsystem" +# "SXM" in the output → needs Fabric Manager +# No "SXM" → PCIe, no Fabric Manager needed +``` -# Download and add NVIDIA GPG key using modern method -wget -qO- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub | gpg --dearmor -o /etc/apt/keyrings/nvidia-cuda.gpg +## Step 2 — Install the GPU Operator -# Add NVIDIA repository with signed-by reference -echo "deb [signed-by=/etc/apt/keyrings/nvidia-cuda.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/ /" | tee /etc/apt/sources.list.d/cuda-repo.list +Add the NVIDIA Helm repo: -apt update +```bash +helm repo add nvidia https://helm.ngc.nvidia.com/nvidia +helm repo update ``` -### Install Driver +Create `gpu-operator-values.yaml`: + +```yaml +operator: + defaultRuntime: containerd -Choose the installation method based on your GPU type: +driver: + enabled: true + rdma: + enabled: false # ← Leave false unless you are doing Part 2 (InfiniBand) + useHostMofed: false -#### Consumer GPUs (RTX 4090, RTX 5090, etc.) +fabricManager: + enabled: false # ← Set true for SXM GPUs (see table above) -For consumer-grade GPUs, install the standard driver: +migManager: + enabled: false + +dcgmExporter: + enabled: true + +nodeStatusExporter: + enabled: true +``` + +> **Why `driver.rdma.enabled: false` by default?** With RDMA enabled, the GPU driver pod's init container waits for MOFED (from the Network Operator) and will sit in `Init:0/1` forever if the Network Operator is not installed. Enable it only in [Part 2](#part-2--infiniband--rdma-optional). + +Deploy: ```bash -DEBIAN_FRONTEND=noninteractive apt -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install nvidia-driver-580 -apt -y autoremove +helm upgrade -i gpu-operator nvidia/gpu-operator \ + --namespace gpu-operator \ + --create-namespace \ + -f gpu-operator-values.yaml ``` -Reboot the node. +Watch the rollout until every pod is `Running` and `nvidia-operator-validator` completes: -#### Data Center GPUs (H100, H200, etc.) +```bash +kubectl -n gpu-operator get pods -w +``` -For data center GPUs (including SXM form factor), install the server driver and Fabric Manager: +## Step 3 — Verify GPUs + +Check allocatable GPUs per node: ```bash -DEBIAN_FRONTEND=noninteractive apt -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install nvidia-driver-580-server -apt-get install nvidia-fabricmanager-580 -systemctl start nvidia-fabricmanager -systemctl enable nvidia-fabricmanager -apt -y autoremove +kubectl get nodes -o custom-columns=\ +NAME:.metadata.name,\ +GPUs:.status.allocatable.nvidia\\.com/gpu ``` -Reboot the node. +Expected: + +``` +NAME GPUs +node1 8 +node2 8 +``` -### Verify Installation +Run a GPU test: ```bash -nvidia-smi +kubectl run gpu-test --rm -it --restart=Never \ + --image=nvidia/cuda:12.4.0-base-ubuntu22.04 \ + --limits=nvidia.com/gpu=1 \ + -- nvidia-smi ``` -You should see your GPUs listed with driver information. +You should see the GPUs listed with driver information. If your nodes have no InfiniBand hardware, you're done — continue to [Next Steps](#next-steps). --- -## STEP 2 - Install NVIDIA Container Toolkit - -Run on **each GPU node**: +# Part 2 — InfiniBand / RDMA (optional) -```bash -# Create keyrings directory if it doesn't exist -mkdir -p /etc/apt/keyrings +> **Only for GPU nodes with Mellanox/NVIDIA ConnectX InfiniBand HCAs.** This section adds the NVIDIA Network Operator (MOFED + the RDMA shared device plugin) and re-enables RDMA in the GPU Operator so multi-node GPU workloads can use the IB fabric for NCCL. If your nodes have no IB hardware, skip this section entirely. -# Download and add NVIDIA Container Toolkit GPG key using modern method -curl -s -L https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /etc/apt/keyrings/nvidia-container-toolkit.gpg +## Step 4 — Verify IB Hardware -# Add NVIDIA Container Toolkit repository with signed-by reference -echo "deb [signed-by=/etc/apt/keyrings/nvidia-container-toolkit.gpg] https://nvidia.github.io/libnvidia-container/stable/deb/amd64 /" | tee /etc/apt/sources.list.d/nvidia-container-toolkit.list +Run on **each GPU+IB node**: -apt-get update -apt-get install -y nvidia-container-toolkit nvidia-container-runtime +```bash +# Confirm no host MOFED is installed (the Network Operator provides its own) +dpkg -l | grep mlnx-ofed +# Should be empty. In-kernel mlx5_core / ib_core modules are fine — the +# Network Operator replaces them with its MOFED versions. + +# Identify IB hardware +lspci | grep -i mellanox +# e.g. 1b:00.0 Infiniband controller: Mellanox Technologies MT28908 [ConnectX-6] + +# Get the PCI device ID (needed for the NicClusterPolicy) +lspci -n | grep 15b3 +# e.g. 1b:00.0 0207: 15b3:101b ← "101b" is the device ID + +# Confirm IB ports are Active +ibstat | grep -E "State|Link layer|Rate" +# Must show: State: Active, Link layer: InfiniBand ``` ---- +### ConnectX Device ID Reference -## STEP 3 - Configure NVIDIA CDI +| Card | PCI device ID | +| --- | --- | +| ConnectX-4 | `1013` | +| ConnectX-5 | `1017` | +| ConnectX-5 Ex | `1019` | +| ConnectX-6 | `101b` | +| ConnectX-6 Dx | `101d` | +| ConnectX-7 | `1021` | +| BlueField-2 | `a2d6` | +| BlueField-3 | `a2dc` | -Run on **each GPU node**: +> **Verify against your own hardware** with `lspci -n | grep 15b3` — the device ID in the `15b3:XXXX` pair is what goes into the NicClusterPolicy `deviceIDs` selector. -### Generate CDI Specification +## Step 5 — Install the Network Operator ```bash -sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml +helm upgrade -i network-operator nvidia/network-operator \ + --namespace nvidia-network-operator \ + --create-namespace \ + --set deployCR=false ``` -### Configure NVIDIA Runtime +`deployCR=false` lets us apply the `NicClusterPolicy` separately in the next step. + +## Step 6 — Apply the NicClusterPolicy -Edit `/etc/nvidia-container-runtime/config.toml` and ensure these lines are **uncommented** and set to: +Create `nic-cluster-policy.yaml`. Replace `` with the ConnectX device ID from Step 4: -```toml -accept-nvidia-visible-devices-as-volume-mounts = false -accept-nvidia-visible-devices-envvar-when-unprivileged = true +```yaml +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy +spec: + ofedDriver: + image: doca-driver + repository: nvcr.io/nvidia/mellanox + version: "doca3.3.0-26.01-1.0.0.0-0" + upgradePolicy: + autoUpgrade: true + maxParallelUpgrades: 1 + safeLoad: false + drain: + enable: true + force: true + timeoutSeconds: 300 + deleteEmptyDir: true + + rdmaSharedDevicePlugin: + image: k8s-rdma-shared-dev-plugin + repository: nvcr.io/nvidia/mellanox + version: "network-operator-v26.1.1" + config: | + { + "configList": [ + { + "resourceName": "rdma_shared_device_ib", + "rdmaHcaMax": 63, + "selectors": { + "vendors": ["15b3"], + "deviceIDs": [""] + } + } + ] + } + + secondaryNetwork: + cniPlugins: + image: plugins + repository: nvcr.io/nvidia/mellanox + version: "network-operator-v26.1.1" + multus: + image: multus-cni + repository: nvcr.io/nvidia/mellanox + version: "network-operator-v26.1.1" + ipoib: + image: ipoib-cni + repository: nvcr.io/nvidia/mellanox + version: "network-operator-v26.1.1" + + nvIpam: + image: nvidia-k8s-ipam + repository: nvcr.io/nvidia/mellanox + version: "network-operator-v26.1.1" + enableWebhook: false ``` -> **Note:** This setup uses **CDI (Container Device Interface)** for device enumeration, which provides better security and device management. +> **Version tags:** the `doca-driver` and `network-operator-v26.1.1` tags above match a specific Network Operator release. Confirm the exact tags for your release in the [NGC catalog](https://catalog.ngc.nvidia.com/) / [Network Operator release notes](https://docs.nvidia.com/networking/display/kubernetes/network-operator) before applying. ---- +**CRD schema notes (Network Operator v26.1.x):** -## STEP 4 - Create NVIDIA RuntimeClass +- `rdmaSharedDevicePlugin` takes a `config` field (a JSON string), **not** a `resources` array. +- `nvIpam` is a **top-level** `spec` field, **not** nested under `secondaryNetwork`. +- `secondaryNetwork` accepts only `cniPlugins`, `multus`, and `ipoib`. +- Component versions should match the operator version (`network-operator-v26.1.1`). -Run from a **control plane node**: +Apply it: ```bash -cat > nvidia-runtime-class.yaml << 'EOF' -kind: RuntimeClass -apiVersion: node.k8s.io/v1 -metadata: - name: nvidia -handler: nvidia -EOF - -kubectl apply -f nvidia-runtime-class.yaml +kubectl apply -f nic-cluster-policy.yaml ``` ---- - -## STEP 5 - Label GPU Nodes - -Label each GPU node (replace `` with actual node name): +Wait for MOFED to compile (5–10 minutes on first deploy) and all pods to reach `Running`: ```bash -kubectl label nodes allow-nvdp=true +kubectl -n nvidia-network-operator get pods -w + +# MOFED compile progress +kubectl -n nvidia-network-operator logs -l app=mofed-ubuntu24.04 -f --tail=10 ``` -### Verify Labels +Expected pods when complete: -```bash -kubectl describe node | grep -A5 Labels +``` +mofed-ubuntu24.04-* 1/1 Running ← one per node, MOFED loaded +rdma-shared-dp-ds-* 1/1 Running ← RDMA device plugin per node +kube-multus-ds-* 1/1 Running +cni-plugins-ds-* 1/1 Running +kube-ipoib-cni-ds-* 1/1 Running +nv-ipam-controller-* 1/1 Running +nv-ipam-node-* 1/1 Running +network-operator-* 1/1 Running ``` -You should see `allow-nvdp=true`. +## Step 7 — Enable RDMA in the GPU Operator ---- +Now that MOFED is available, re-enable RDMA in the GPU Operator so the driver loads `nvidia-peermem` for GPUDirect RDMA. Edit `gpu-operator-values.yaml`: -## STEP 6 - Install NVIDIA Device Plugin +```yaml +driver: + enabled: true + rdma: + enabled: true # ← Now true; MOFED is present + useHostMofed: false +``` -Run from a **control plane node**: +Re-apply: ```bash -helm repo add nvdp https://nvidia.github.io/k8s-device-plugin -helm repo update - -helm upgrade -i nvdp nvdp/nvidia-device-plugin \ - --namespace nvidia-device-plugin \ - --create-namespace \ - --version 0.18.0 \ - --set runtimeClassName="nvidia" \ - --set deviceListStrategy=cdi-cri \ - --set nvidiaDriverRoot="/" \ - --set-string nodeSelector.allow-nvdp="true" +helm upgrade -i gpu-operator nvidia/gpu-operator \ + --namespace gpu-operator \ + -f gpu-operator-values.yaml ``` -### Verify Installation +The GPU driver daemonset rolls; the driver init container detects MOFED and loads `nvidia-peermem`. Wait for the driver pods and `nvidia-operator-validator` to return to `Running`/complete: ```bash -kubectl -n nvidia-device-plugin get pods -o wide +kubectl -n gpu-operator get pods -w ``` -You should see `nvdp-nvidia-device-plugin` pods running on your GPU nodes. +## Step 8 — Verify RDMA -### Check Logs +Check allocatable RDMA devices per node (`rdmaHcaMax` was `63`): ```bash -kubectl -n nvidia-device-plugin logs -l app.kubernetes.io/instance=nvdp +kubectl get nodes -o json | \ + jq -r '.items[] | "\(.metadata.name) rdma=\(.status.allocatable["rdma/rdma_shared_device_ib"] // "none")"' ``` -**Expected output:** +Expected: ``` -Detected NVML platform: found NVML library -Starting GRPC server for 'nvidia.com/gpu' -Registered device plugin for 'nvidia.com/gpu' with Kubelet +node1 rdma=63 +node2 rdma=63 ``` ---- - -## STEP 7 - Test GPU Functionality +### Cross-Node Bandwidth Test -Create a test pod: +Server on `node1`: ```bash -cat > gpu-test-pod.yaml << 'EOF' -apiVersion: v1 -kind: Pod -metadata: - name: gpu-test -spec: - restartPolicy: Never - runtimeClassName: nvidia - containers: - - name: cuda-container - image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.6.0 - resources: - limits: - nvidia.com/gpu: 1 - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule -EOF - -kubectl apply -f gpu-test-pod.yaml +kubectl run ib-server --image=mellanox/rping-test --restart=Never \ + --overrides='{ + "spec": { + "nodeName": "node1", + "containers": [{ + "name": "s", + "image": "mellanox/rping-test", + "command": ["ib_write_bw", "-d", "mlx5_0", "--report_gbits"], + "resources": {"limits": {"rdma/rdma_shared_device_ib": "1"}}, + "securityContext": {"capabilities": {"add": ["IPC_LOCK"]}} + }] + } + }' ``` -### Verify Test - -Wait for the pod to complete, then check logs: +Client on `node2`: ```bash -kubectl logs gpu-test +SERVER_IP=$(kubectl get pod ib-server -o jsonpath='{.status.podIP}') + +kubectl run ib-client --image=mellanox/rping-test --restart=Never \ + --overrides="{ + \"spec\": { + \"nodeName\": \"node2\", + \"containers\": [{ + \"name\": \"c\", + \"image\": \"mellanox/rping-test\", + \"command\": [\"ib_write_bw\", \"-d\", \"mlx5_0\", \"$SERVER_IP\", \"--report_gbits\"], + \"resources\": {\"limits\": {\"rdma/rdma_shared_device_ib\": \"1\"}}, + \"securityContext\": {\"capabilities\": {\"add\": [\"IPC_LOCK\"]}} + }] + } + }" + +kubectl logs ib-client +# Approximate full-port line rates: ~197 Gb/s (HDR), ~100 Gb/s (HDR100 / EDR) + +kubectl delete pod ib-server ib-client ``` -**Expected output:** +> **NCCL configuration is handled by the Akash provider.** You do not configure NCCL on the cluster. When a tenant requests GPU interconnect in the SDL, the provider auto-injects the NCCL environment (`NCCL_IB_DISABLE=0`, `NCCL_IB_HCA` from the node's discovered HCA families, `NCCL_IB_GID_INDEX=3` on RoCE) and requests one `rdma/rdma_shared_device_ib` handle per GPU. Provider setup ends once the node advertises the GPU and RDMA resources verified above. + +--- + +## Deployment Order Summary ``` -[Vector addition of 50000 elements] -Copy input data from the host memory to the CUDA device -CUDA kernel launch with 196 blocks of 256 threads -Copy output data from the CUDA device to the host memory -Test PASSED -Done +GPU-only provider: + 1. Kubespray (kubeadm + Calico + containerd) + 2. GPU Operator (driver.rdma.enabled: false) + 3. Nodes report nvidia.com/gpu + +Adding InfiniBand: + 4. Network Operator (helm) + 5. NicClusterPolicy → MOFED compile + RDMA device plugin + ↓ wait for MOFED pods Running + 6. GPU Operator helm upgrade (driver.rdma.enabled: true) + ↓ driver reloads, loads nvidia-peermem + 7. Nodes report nvidia.com/gpu AND rdma/rdma_shared_device_ib ``` -### Test nvidia-smi +--- + +## Troubleshooting -Create an interactive test: +**GPU driver pod stuck in `Init:0/1`** — the driver init container is waiting for MOFED. This happens when `driver.rdma.enabled: true` but the Network Operator/MOFED is not ready. Either finish Part 2 or set `driver.rdma.enabled: false`. ```bash -kubectl run gpu-shell --rm -it --restart=Never --image=nvidia/cuda:11.6.2-base-ubuntu20.04 -- nvidia-smi +kubectl -n nvidia-network-operator get pods | grep mofed +kubectl -n nvidia-network-operator logs -l app=mofed-ubuntu24.04 --tail=20 ``` -You should see GPU information displayed. +**NicClusterPolicy rejected with schema errors** — the CRD schema changes between operator versions. Inspect what your version accepts: + +```bash +# Top-level spec fields +kubectl get crd nicclusterpolicies.mellanox.com -o json | \ + jq '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties | keys' + +# secondaryNetwork sub-fields +kubectl get crd nicclusterpolicies.mellanox.com -o json | \ + jq '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.secondaryNetwork.properties | keys' +``` -### Cleanup +**RDMA resources show 0** — check the device plugin logs and confirm `deviceIDs` matches your hardware: ```bash -kubectl delete pod gpu-test +kubectl -n nvidia-network-operator logs -l app=rdma-shared-dp --tail=20 +lspci -n | grep 15b3 ``` +**IPoIB CNI errors during bring-up** — the NicClusterPolicy deploys an IPoIB CNI, but RDMA verbs traffic does not use it — it goes directly over the HCA, bypassing the kernel network stack. IPoIB errors in `dmesg` do not affect RDMA and won't show up in the `ib_write_bw` verification above. + --- ## Next Steps -Your Kubernetes cluster now has GPU support! +Your cluster now has GPU support (and optionally InfiniBand RDMA). -**Optional enhancements:** -- [Provider installation – STEP 9 (TLS)](/docs/providers/setup-and-installation/kubespray/provider-installation-prep#step-9---lets-encrypt-cert-manager-and-tls-secrets) - **Required** for all providers: cert-manager and Gateway TLS -- [IP Leases](/docs/providers/setup-and-installation/kubespray/ip-leases) - Enable static IPs +- [Provider installation – STEP 9 (TLS)](/docs/providers/setup-and-installation/kubespray/provider-installation-prep#step-9---lets-encrypt-cert-manager-and-tls-secrets) — **required** for all providers: cert-manager and Gateway TLS +- [IP Leases](/docs/providers/setup-and-installation/kubespray/ip-leases) — enable static IPs -> **Note:** After installing the provider, you'll need to add GPU attributes to your `provider.yaml` to advertise GPU capabilities. This is covered in the Provider Installation guide. +> **Note:** After installing the provider, add GPU attributes to your `provider.yaml` to advertise GPU capabilities (and `capabilities/gpu-interconnect` for InfiniBand). This is covered in the Provider Installation guide. From a71503992e484626e6a40ff8dc86f2362a18f8b8 Mon Sep 17 00:00:00 2001 From: zblocker64 Date: Fri, 17 Jul 2026 10:27:32 -0500 Subject: [PATCH 2/4] chore: add attributes --- .../operations/provider-attributes/index.md | 62 +++++++++++++++++++ .../operations/provider-audit/index.md | 14 ++++- .../kubespray/gpu-support/index.md | 3 +- .../provider-installation-prep/index.mdx | 16 +++++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/content/Docs/providers/operations/provider-attributes/index.md b/src/content/Docs/providers/operations/provider-attributes/index.md index 1eeeefd2c..b6597dab0 100644 --- a/src/content/Docs/providers/operations/provider-attributes/index.md +++ b/src/content/Docs/providers/operations/provider-attributes/index.md @@ -268,6 +268,19 @@ See [GPU Capabilities](#gpu-capabilities) below for RAM and interface sub-keys. value: "12.7" ``` +### capabilities/gpu-interconnect + +- **Value**: `"true"` +- **Purpose**: Advertise multi-node GPU interconnect (RDMA over InfiniBand or RoCE) for NCCL workloads +- **Required**: Only if you completed [InfiniBand / RDMA setup](/docs/providers/setup-and-installation/kubespray/gpu-support#part-2--infiniband--rdma-optional) + +```yaml +- key: capabilities/gpu-interconnect + value: "true" +``` + +Also advertise the fabric you offer — see [GPU Interconnect](#gpu-interconnect-infiniband--roce) below. + ### capabilities/memory - **Values**: `ddr2`, `ddr3`, `ddr3ecc`, `ddr4`, `ddr4ecc`, `ddr5`, `ddr5ecc` @@ -551,6 +564,49 @@ If your GPU isn't in the [provider-configs database](https://github.com/akash-ne 3. Wait for approval (typically 1–3 business days) 4. Update your provider attributes with the approved capability keys +### GPU Interconnect (InfiniBand / RoCE) + +Advertise these attributes only after you have completed [Part 2 — InfiniBand / RDMA](/docs/providers/setup-and-installation/kubespray/gpu-support#part-2--infiniband--rdma-optional) and verified RDMA on the cluster. Tenants request interconnect in the SDL with `gpu.attributes.interconnect` and match on these placement attributes. + +| On-chain key | When to set | +| ------------ | ----------- | +| `capabilities/gpu-interconnect` | Always, if you offer multi-node RDMA | +| `capabilities/gpu-interconnect/fabric/infiniband` | If nodes use InfiniBand HCAs | +| `capabilities/gpu-interconnect/fabric/roce` | If nodes use RoCE | + +Advertise every fabric you actually support. Omit all interconnect keys if you have no InfiniBand/RoCE hardware. + +**InfiniBand example:** + +```yaml +- key: capabilities/gpu-interconnect + value: "true" +- key: capabilities/gpu-interconnect/fabric/infiniband + value: "true" +``` + +**RoCE example:** + +```yaml +- key: capabilities/gpu-interconnect + value: "true" +- key: capabilities/gpu-interconnect/fabric/roce + value: "true" +``` + +**Both fabrics:** + +```yaml +- key: capabilities/gpu-interconnect + value: "true" +- key: capabilities/gpu-interconnect/fabric/infiniband + value: "true" +- key: capabilities/gpu-interconnect/fabric/roce + value: "true" +``` + +> **Note:** Tenants can pin InfiniBand or RoCE with `capabilities/gpu-interconnect/fabric/infiniband` or `.../fabric/roce` in the placement attributes. Without a fabric pin, any provider advertising `capabilities/gpu-interconnect` can bid. See [GPU Interconnect (multi-node RDMA)](/docs/learn/core-concepts/gpu-deployments/#gpu-interconnect-multi-node-rdma). + --- ## Complete Example @@ -638,6 +694,12 @@ attributes: value: "true" - key: cuda value: "12.7" + + # GPU Interconnect (only if you completed InfiniBand / RDMA setup) + # - key: capabilities/gpu-interconnect + # value: "true" + # - key: capabilities/gpu-interconnect/fabric/infiniband + # value: "true" ``` --- diff --git a/src/content/Docs/providers/operations/provider-audit/index.md b/src/content/Docs/providers/operations/provider-audit/index.md index bc268f8f8..22aaa111f 100644 --- a/src/content/Docs/providers/operations/provider-audit/index.md +++ b/src/content/Docs/providers/operations/provider-audit/index.md @@ -141,9 +141,21 @@ See [Provider Attributes — GPU Capabilities](/docs/providers/operations/provid If you **do not** run GPUs, omit all `capabilities/gpu/*` keys and `cuda`. +### Required if you offer GPU interconnect (InfiniBand / RoCE) + +If you advertise multi-node RDMA, include: + +| On-chain key | Notes | +| ------------ | ----- | +| `capabilities/gpu-interconnect` | `"true"` | +| `capabilities/gpu-interconnect/fabric/infiniband` | If you offer InfiniBand | +| `capabilities/gpu-interconnect/fabric/roce` | If you offer RoCE | + +See [Provider Attributes — GPU Interconnect](/docs/providers/operations/provider-attributes/#gpu-interconnect-infiniband--roce). Omit these keys if you have no InfiniBand/RoCE fabric. + ### Attributes must match what you offer -On-chain attributes must match **every capability you advertise** — GPU models, SHM, persistent storage, IP leases, custom domains, and network speeds. Reviewers compare your attributes to the audit deployment and your live inventory. +On-chain attributes must match **every capability you advertise** — GPU models, GPU interconnect, SHM, persistent storage, IP leases, custom domains, and network speeds. Reviewers compare your attributes to the audit deployment and your live inventory. --- diff --git a/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md b/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md index a5ab79ca1..81acbd147 100644 --- a/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md +++ b/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md @@ -453,4 +453,5 @@ Your cluster now has GPU support (and optionally InfiniBand RDMA). - [Provider installation – STEP 9 (TLS)](/docs/providers/setup-and-installation/kubespray/provider-installation-prep#step-9---lets-encrypt-cert-manager-and-tls-secrets) — **required** for all providers: cert-manager and Gateway TLS - [IP Leases](/docs/providers/setup-and-installation/kubespray/ip-leases) — enable static IPs -> **Note:** After installing the provider, add GPU attributes to your `provider.yaml` to advertise GPU capabilities (and `capabilities/gpu-interconnect` for InfiniBand). This is covered in the Provider Installation guide. +> **Note:** After installing the provider, add GPU attributes to your `provider.yaml` to advertise GPU capabilities — and `capabilities/gpu-interconnect` plus `capabilities/gpu-interconnect/fabric/infiniband` (or `.../fabric/roce`) if you completed Part 2. See [Provider Attributes — GPU Interconnect](/docs/providers/operations/provider-attributes/#gpu-interconnect-infiniband--roce). + diff --git a/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx b/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx index 10ec45b0f..e0e92b2be 100644 --- a/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx +++ b/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx @@ -321,6 +321,22 @@ attributes: > **Note:** Model names should be lowercase with no spaces. List each GPU model you're offering. +### Add GPU Interconnect Attributes (if you have InfiniBand / RoCE) + +If you completed [Part 2 — InfiniBand / RDMA](/docs/providers/setup-and-installation/kubespray/gpu-support#part-2--infiniband--rdma-optional), advertise interconnect so tenants can match multi-node NCCL jobs: + +```yaml + - key: capabilities/gpu-interconnect + value: "true" + - key: capabilities/gpu-interconnect/fabric/infiniband + value: "true" + # Or RoCE instead of (or in addition to) InfiniBand: + # - key: capabilities/gpu-interconnect/fabric/roce + # value: "true" +``` + +Omit these keys if your nodes have no InfiniBand/RoCE hardware. Full details: [Provider Attributes — GPU Interconnect](/docs/providers/operations/provider-attributes/#gpu-interconnect-infiniband--roce). + --- ### Complete Example with All Features From e8de3bff63d360c232b560248b866f35a8ab13f4 Mon Sep 17 00:00:00 2001 From: zblocker64 Date: Fri, 17 Jul 2026 10:33:17 -0500 Subject: [PATCH 3/4] chore: more fixes --- .../architecture/operators/inventory/index.md | 3 +- .../hardware-requirements/index.md | 14 ++-- .../gpu-operator-migration/index.md | 9 +- .../operations/provider-attributes/index.md | 6 +- .../kubespray/gpu-support/index.md | 70 +++++++++++++--- .../kubespray/persistent-storage/index.md | 83 ++++++++++--------- .../provider-installation-prep/index.mdx | 2 +- 7 files changed, 120 insertions(+), 67 deletions(-) diff --git a/src/content/Docs/providers/architecture/operators/inventory/index.md b/src/content/Docs/providers/architecture/operators/inventory/index.md index 17310e4f1..6f5535e45 100644 --- a/src/content/Docs/providers/architecture/operators/inventory/index.md +++ b/src/content/Docs/providers/architecture/operators/inventory/index.md @@ -133,7 +133,7 @@ memory: ```yaml storage: - - class: beta2 # Storage class name + - class: beta3 # Storage class name (NVMe default) size: 5000Gi # Total capacity provisioner: ceph.rook.io ``` @@ -246,7 +246,6 @@ The operator adds labels to nodes based on discovered hardware: ```yaml akash.network/capabilities.gpu.vendor.nvidia: "true" akash.network/capabilities.gpu.model.rtx4090: "true" -akash.network/capabilities.storage.class.beta2: "true" akash.network/capabilities.storage.class.beta3: "true" ``` diff --git a/src/content/Docs/providers/getting-started/hardware-requirements/index.md b/src/content/Docs/providers/getting-started/hardware-requirements/index.md index 13092c2a4..f9e750a36 100644 --- a/src/content/Docs/providers/getting-started/hardware-requirements/index.md +++ b/src/content/Docs/providers/getting-started/hardware-requirements/index.md @@ -113,8 +113,8 @@ This guide covers system requirements, hardware specifications, and best practic ### Persistent Storage (Optional) **Minimum Requirements:** -- **Option 1:** 4 SSDs across all nodes (minimum) -- **Option 2:** 2 NVMe SSDs across all nodes (minimum) +- **Recommended (default in [Persistent Storage](/docs/providers/setup-and-installation/kubespray/persistent-storage)):** 2+ NVMe SSDs across all nodes (`beta3`) +- **Alternative:** 4 SATA/SAS SSDs across all nodes (`beta2`) - All persistent storage must be **same type** across entire cluster - **Do not mix** storage types (e.g., SSD + NVMe, or beta2 + beta3) @@ -124,14 +124,14 @@ This guide covers system requirements, hardware specifications, and best practic - **Recommended:** Distribute dedicated drives across multiple nodes for redundancy **Storage Classes:** -- `beta1` - HDD (Hard Disk Drive) +- `beta3` - NVMe (Non-Volatile Memory Express) — **default** - `beta2` - SSD (Solid State Drive) -- `beta3` - NVMe (Non-Volatile Memory Express) +- `beta1` - HDD (Hard Disk Drive) **Example Configurations:** -- **Minimum SSD:** 4x 1TB SSDs across 2-4 nodes (beta2) -- **Minimum NVMe:** 2x 2TB NVMe drives across 2 nodes (beta3) -- **Recommended:** 6+ drives distributed across 3+ nodes for better redundancy +- **Minimum NVMe (recommended):** 2x 2TB NVMe drives across 2 nodes (`beta3`) +- **Minimum SSD:** 4x 1TB SSDs across 2-4 nodes (`beta2`) +- **Recommended:** 6+ NVMe drives distributed across 3+ nodes for better redundancy --- diff --git a/src/content/Docs/providers/operations/gpu-operator-migration/index.md b/src/content/Docs/providers/operations/gpu-operator-migration/index.md index 42d99863e..de0f94b6a 100644 --- a/src/content/Docs/providers/operations/gpu-operator-migration/index.md +++ b/src/content/Docs/providers/operations/gpu-operator-migration/index.md @@ -122,15 +122,17 @@ After reboot, `nvidia-smi` on the host should report **"command not found"** or ## Step 4 — Install the GPU Operator -Once your GPU nodes are clean, install the Operator cluster-wide. Use the same values as a fresh setup — see [GPU & InfiniBand Support → Step 2](/docs/providers/setup-and-installation/kubespray/gpu-support#step-2--install-the-gpu-operator) for the full file and the Fabric Manager guidance: +Once your GPU nodes are clean, install the Operator cluster-wide. Use the same values as a fresh setup — see [GPU & InfiniBand Support → Step 2](/docs/providers/setup-and-installation/kubespray/gpu-support#step-2--install-the-gpu-operator) for the full file, Fabric Manager guidance, CRD notes, and pinned chart version (`v26.3.3`). ```bash helm repo add nvidia https://helm.ngc.nvidia.com/nvidia helm repo update -helm upgrade -i gpu-operator nvidia/gpu-operator \ +# First deploy: helm install (not upgrade -i) so chart CRDs are applied +helm install gpu-operator nvidia/gpu-operator \ --namespace gpu-operator \ --create-namespace \ + --version v26.3.3 \ -f gpu-operator-values.yaml ``` @@ -145,12 +147,13 @@ fabricManager: enabled: false # Set true for SXM GPUs ``` -Watch the rollout until every pod is `Running` and `nvidia-operator-validator` completes: +Watch the rollout until every pod is `Running` and validators succeed (`nvidia-cuda-validator` = `Completed`, `nvidia-operator-validator` = `Running`): ```bash kubectl -n gpu-operator get pods -w ``` +Brief driver crashloops during first bring-up often self-heal — wait for the validators before rolling back. The Operator's driver daemonset only schedules on nodes it can manage — cleaned, uncordoned nodes. Uncordon each node after its driver pod is ready (Step 5). ## Step 5 — Uncordon and verify diff --git a/src/content/Docs/providers/operations/provider-attributes/index.md b/src/content/Docs/providers/operations/provider-attributes/index.md index b6597dab0..9eaf42d7b 100644 --- a/src/content/Docs/providers/operations/provider-attributes/index.md +++ b/src/content/Docs/providers/operations/provider-attributes/index.md @@ -406,7 +406,7 @@ Advertise support for advanced features. value: "true" ``` -> **Important:** You can only advertise **one persistent storage class** per provider. Choose either beta1 (HDD), beta2 (SSD), or beta3 (NVMe) based on what you configured in Rook-Ceph. +> **Important:** You can only advertise **one persistent storage class** per provider. Match Rook-Ceph: **`beta3` (NVMe) is the default** in the Persistent Storage guide; use `beta2` (SSD) or `beta1` (HDD) only if you configured that instead. ### feat-shm @@ -671,9 +671,9 @@ attributes: - key: feat-endpoint-custom-domain value: true - # Persistent Storage (required if you have Rook-Ceph) + # Persistent Storage (required if you have Rook-Ceph; beta3 = NVMe default) - key: capabilities/storage/1/class - value: beta2 + value: beta3 - key: capabilities/storage/1/persistent value: "true" diff --git a/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md b/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md index 81acbd147..8d4412a61 100644 --- a/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md +++ b/src/content/Docs/providers/setup-and-installation/kubespray/gpu-support/index.md @@ -78,6 +78,8 @@ helm repo add nvidia https://helm.ngc.nvidia.com/nvidia helm repo update ``` +Pin the chart to the current stable patch (**`v26.3.3`**). NVIDIA GPU Operator charts use CalVer (`vYY.MM.PP`). + Create `gpu-operator-values.yaml`: ```yaml @@ -105,21 +107,34 @@ nodeStatusExporter: > **Why `driver.rdma.enabled: false` by default?** With RDMA enabled, the GPU driver pod's init container waits for MOFED (from the Network Operator) and will sit in `Init:0/1` forever if the Network Operator is not installed. Enable it only in [Part 2](#part-2--infiniband--rdma-optional). -Deploy: +### First deploy — CRDs + +`helm upgrade --install` **skips** a chart's `crds/` directory. On a first deploy that causes failures like `no matches for kind "ClusterPolicy"`. Use `helm install` for the initial release (or pre-apply CRDs — see [Troubleshooting](#troubleshooting)). + +Deploy (first install): ```bash -helm upgrade -i gpu-operator nvidia/gpu-operator \ +helm install gpu-operator nvidia/gpu-operator \ --namespace gpu-operator \ --create-namespace \ + --version v26.3.3 \ -f gpu-operator-values.yaml ``` -Watch the rollout until every pod is `Running` and `nvidia-operator-validator` completes: +Later upgrades can use `helm upgrade` with the same `--version` and values file. + +Watch the rollout until every pod is `Running` and validators succeed: ```bash kubectl -n gpu-operator get pods -w + +# Healthy end state +kubectl -n gpu-operator get pods | grep -E 'cuda-validator|operator-validator' +# nvidia-cuda-validator-* Completed +# nvidia-operator-validator-* Running ``` +> **Note:** Driver pods may crashloop briefly during the first bring-up while dependencies start, then self-heal. Wait for the validators above before treating it as a failure. ## Step 3 — Verify GPUs Check allocatable GPUs per node: @@ -195,18 +210,29 @@ ibstat | grep -E "State|Link layer|Rate" ## Step 5 — Install the Network Operator +Pin the chart to **`26.1.1`** (CalVer `YY.MM.PP`). Unpinned installs can pull a newer chart (for example `26.4.0`) whose CRDs/images do not match the NicClusterPolicy below. + +Same CRD rule as the GPU Operator: use `helm install` on first deploy (or pre-apply CRDs). `helm upgrade --install` skips `crds/` and fails with `no matches for kind "NodeFeatureRule"`. + ```bash -helm upgrade -i network-operator nvidia/network-operator \ +helm install network-operator nvidia/network-operator \ --namespace nvidia-network-operator \ --create-namespace \ - --set deployCR=false + --version 26.1.1 \ + --set deployCR=false \ + --set nfd.enabled=true ``` -`deployCR=false` lets us apply the `NicClusterPolicy` separately in the next step. +- `deployCR=false` — apply the `NicClusterPolicy` yourself in the next step. +- `nfd.enabled=true` — Node Feature Discovery (required; the chart registers `NodeFeatureRule` CRDs). + +Later upgrades: `helm upgrade network-operator nvidia/network-operator --version 26.1.1 ...` with the same flags. ## Step 6 — Apply the NicClusterPolicy -Create `nic-cluster-policy.yaml`. Replace `` with the ConnectX device ID from Step 4: +Create `nic-cluster-policy.yaml`. Set `deviceIDs` from Step 4 (`lspci -n | grep 15b3`). Example below uses **`101b`** (ConnectX-6); change it if your cards differ. + +Component image tags are pinned to **Network Operator v26.1.1** (`network-operator-v26.1.1` and DOCA driver `doca3.3.0-26.01-1.0.0.0-0`). Keep them aligned with the chart version from Step 5. ```yaml apiVersion: mellanox.com/v1alpha1 @@ -240,7 +266,7 @@ spec: "rdmaHcaMax": 63, "selectors": { "vendors": ["15b3"], - "deviceIDs": [""] + "deviceIDs": ["101b"] } } ] @@ -267,11 +293,12 @@ spec: enableWebhook: false ``` -> **Version tags:** the `doca-driver` and `network-operator-v26.1.1` tags above match a specific Network Operator release. Confirm the exact tags for your release in the [NGC catalog](https://catalog.ngc.nvidia.com/) / [Network Operator release notes](https://docs.nvidia.com/networking/display/kubernetes/network-operator) before applying. +> **Version tags:** the `doca-driver` and `network-operator-v26.1.1` tags above match Network Operator chart `26.1.1`. If you change the chart version, update these tags from the [NGC catalog](https://catalog.ngc.nvidia.com/) / [Network Operator release notes](https://docs.nvidia.com/networking/display/kubernetes/network-operator). **CRD schema notes (Network Operator v26.1.x):** - `rdmaSharedDevicePlugin` takes a `config` field (a JSON string), **not** a `resources` array. +- Resource name must be `rdma_shared_device_ib` with `rdmaHcaMax: 63` (Akash interconnect expects this resource). - `nvIpam` is a **top-level** `spec` field, **not** nested under `secondaryNetwork`. - `secondaryNetwork` accepts only `cniPlugins`, `multus`, and `ipoib`. - Component versions should match the operator version (`network-operator-v26.1.1`). @@ -316,11 +343,12 @@ driver: useHostMofed: false ``` -Re-apply: +Re-apply (chart already installed — `helm upgrade` is correct here; CRDs are already present): ```bash -helm upgrade -i gpu-operator nvidia/gpu-operator \ +helm upgrade gpu-operator nvidia/gpu-operator \ --namespace gpu-operator \ + --version v26.3.3 \ -f gpu-operator-values.yaml ``` @@ -416,6 +444,24 @@ Adding InfiniBand: ## Troubleshooting +**`no matches for kind "ClusterPolicy"` or `"NodeFeatureRule"`** — Helm skipped CRDs because the release was created with `helm upgrade --install`. On first deploy use `helm install`, **or** pre-apply CRDs then upgrade: + +```bash +# GPU Operator CRDs +helm pull nvidia/gpu-operator --version v26.3.3 --untar +kubectl create -f gpu-operator/crds/ +helm upgrade -i gpu-operator nvidia/gpu-operator \ + --namespace gpu-operator --create-namespace \ + --version v26.3.3 -f gpu-operator-values.yaml + +# Network Operator CRDs +helm pull nvidia/network-operator --version 26.1.1 --untar +kubectl create -f network-operator/crds/ +helm upgrade -i network-operator nvidia/network-operator \ + --namespace nvidia-network-operator --create-namespace \ + --version 26.1.1 --set deployCR=false --set nfd.enabled=true +``` + **GPU driver pod stuck in `Init:0/1`** — the driver init container is waiting for MOFED. This happens when `driver.rdma.enabled: true` but the Network Operator/MOFED is not ready. Either finish Part 2 or set `driver.rdma.enabled: false`. ```bash @@ -423,6 +469,8 @@ kubectl -n nvidia-network-operator get pods | grep mofed kubectl -n nvidia-network-operator logs -l app=mofed-ubuntu24.04 --tail=20 ``` +**Driver pods crashloop then recover** — common during first bring-up while the toolkit / device plugin / MOFED ordering settles. Wait for `nvidia-cuda-validator` = `Completed` and `nvidia-operator-validator` = `Running` before debugging further. + **NicClusterPolicy rejected with schema errors** — the CRD schema changes between operator versions. Inspect what your version accepts: ```bash diff --git a/src/content/Docs/providers/setup-and-installation/kubespray/persistent-storage/index.md b/src/content/Docs/providers/setup-and-installation/kubespray/persistent-storage/index.md index 48129a530..48668468c 100644 --- a/src/content/Docs/providers/setup-and-installation/kubespray/persistent-storage/index.md +++ b/src/content/Docs/providers/setup-and-installation/kubespray/persistent-storage/index.md @@ -9,7 +9,7 @@ description: "Enable persistent storage on your Akash provider using Rook-Ceph" > **Don't need persistent storage?** Skip to [Provider Installation](/docs/providers/setup-and-installation/kubespray/provider-installation). -This guide shows how to enable persistent storage on your Akash provider using Rook-Ceph. +This guide shows how to enable persistent storage on your Akash provider using Rook-Ceph. The default configuration targets **NVMe** drives and the **`beta3`** storage class. **Time:** 45-60 minutes @@ -22,7 +22,8 @@ Before starting, ensure you have: ### Hardware Requirements See [Hardware Requirements - Persistent Storage](/docs/providers/getting-started/hardware-requirements#persistent-storage-optional) for detailed specifications: -- **Minimum:** 4 SSDs across all nodes, OR 2 NVMe SSDs across all nodes +- **Recommended (default in this guide):** NVMe drives — minimum 2 NVMe SSDs across all nodes +- **Alternative:** 4 SATA/SAS SSDs across all nodes (use `beta2` / `osdsPerDevice: "1"` instead) - **Drives must be:** - Dedicated exclusively to persistent storage - Unformatted (no partitions or filesystems) @@ -45,7 +46,7 @@ For production use: **OSDs per drive:** - HDD: 1 OSD max - SSD: 1 OSD max -- NVMe: 2 OSDs max +- NVMe: 2 OSDs max (default in this guide) > **Important:** Do NOT run multiple OSDs on a single SAS/SATA drive. NVMe drives can achieve improved performance with 2 OSDs per drive. @@ -67,16 +68,17 @@ SSH into each potential storage node and list unformatted drives: lsblk -f ``` -Look for drives with no `FSTYPE` (unformatted). Example output: +Look for drives with no `FSTYPE` (unformatted). Example NVMe output: ``` -NAME FSTYPE LABEL UUID MOUNTPOINT -sda -sdb -sdc ext4 a1b2c3d4-5678-90ab-cdef-1234567890ab / +NAME FSTYPE LABEL UUID MOUNTPOINT +nvme0n1 +nvme1n1 +nvme2n1 +nvme0n2 ext4 a1b2c3d4-5678-90ab-cdef-1234567890ab / ``` -In this example, `sda` and `sdb` are unformatted and can be used for Ceph. +In this example, `nvme0n1`, `nvme1n1`, and `nvme2n1` are unformatted and can be used for Ceph. Do not use the OS disk (here `nvme0n2`). ### Wipe Drives (if needed) @@ -84,8 +86,8 @@ If drives have existing partitions or filesystems, wipe them: ```bash # WARNING: This destroys all data on the drive! -sudo wipefs -a /dev/sda -sudo wipefs -a /dev/sdb +sudo wipefs -a /dev/nvme0n1 +sudo wipefs -a /dev/nvme1n1 ``` --- @@ -153,7 +155,7 @@ rook-discover-yyy 1/1 Running 0 2m ### Create Cluster Configuration -Create a file with your storage node and device information: +Create a file with your storage node and device information. The defaults below are for **NVMe** (`beta3`): ```bash cat > rook-ceph-cluster.values.yml << 'EOF' @@ -177,11 +179,11 @@ cephClusterSpec: storage: useAllNodes: false useAllDevices: false - deviceFilter: "^sd[ab]" # Adjust to match your devices + deviceFilter: "^nvme[0-9]+n1$" # All NVMe namespaces nvme0n1, nvme1n1, … config: - osdsPerDevice: "1" # Set to "2" for NVMe drives + osdsPerDevice: "2" # NVMe default; use "1" for SATA/SAS SSD or HDD nodes: - - name: "node1" # Replace with your actual node names + - name: "node1" # Replace with your actual node names - name: "node2" - name: "node3" @@ -196,7 +198,7 @@ cephBlockPools: bulk: "true" storageClass: enabled: true - name: beta2 # SSD storage class + name: beta3 # NVMe storage class (default) isDefault: true reclaimPolicy: Delete allowVolumeExpansion: true @@ -226,17 +228,18 @@ EOF - **dataDirHostPath:** Default is `/var/lib/rook` - If using a custom mount point (e.g., RAID array at `/data`), change to `/data/rook` - This directory stores Ceph monitor and manager data (not OSD data) -- **deviceFilter:** Adjust to match your drives - - SATA/SAS drives: `"^sd[ab]"` - - NVMe drives: `"^nvme[01]n1"` -- **osdsPerDevice:** - - NVMe drives: `"2"` - - HDD/SSD drives: `"1"` +- **deviceFilter:** Must match your data drives only (never the OS disk) + - **NVMe (default):** `"^nvme[0-9]+n1$"` — matches `nvme0n1`, `nvme1n1`, `nvme10n1`, etc. + - Do **not** use `"^nvme[01]n1"` — that is a single-character class (`0` or `1` only), so drives like `nvme2n1` are skipped and you get far fewer OSDs than expected + - SATA/SAS SSD or HDD: `"^sd[a-z]+$"` (adjust to your device names) +- **osdsPerDevice:** + - NVMe (default): `"2"` + - HDD/SSD: `"1"` - **nodes:** Replace with your actual storage node names from `kubectl get nodes` -- **storageClass name:** - - HDD: `beta1` +- **storageClass name:** + - NVMe (default): `beta3` - SSD: `beta2` - - NVMe: `beta3` + - HDD: `beta1` ### Install Cluster @@ -274,7 +277,7 @@ Wait until `PHASE` is `Ready` and `HEALTH` is `HEALTH_OK`. kubectl -n rook-ceph get pods -l app=rook-ceph-osd ``` -You should see OSD pods running on your storage nodes. +You should see OSD pods running on your storage nodes. With `osdsPerDevice: "2"`, expect **two OSDs per NVMe drive** matched by `deviceFilter`. ### Check Ceph Status @@ -297,6 +300,8 @@ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph status osd: 6 osds: 6 up, 6 in ``` +(OSD count depends on how many NVMe drives match the filter × `osdsPerDevice`.) + ### Check Storage Class ```bash @@ -307,29 +312,23 @@ kubectl get storageclass ``` NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE AGE -beta2 (default) rook-ceph.rbd.csi.ceph.com Delete Immediate 5m +beta3 (default) rook-ceph.rbd.csi.ceph.com Delete Immediate 5m ``` ### Label Storage Class for Akash -Label your storage class so Akash can recognize it. Replace `` with the actual name of your storage class (e.g., `beta1` for HDD, `beta2` for SSD, or `beta3` for NVMe): +Label your storage class so Akash can recognize it. With the defaults in this guide that is `beta3`: ```bash -kubectl label sc akash.network=true +kubectl label sc beta3 akash.network=true ``` -**Example:** - -If you configured an SSD storage class named `beta2`: - -```bash -kubectl label sc beta2 akash.network=true -``` +If you used `beta1` or `beta2` instead, label that class name. Verify the label was applied: ```bash -kubectl get sc --show-labels +kubectl get sc beta3 --show-labels ``` You should see `akash.network=true` in the labels. @@ -352,7 +351,7 @@ spec: resources: requests: storage: 1Gi - storageClassName: beta2 + storageClassName: beta3 EOF kubectl apply -f test-pvc.yaml @@ -368,7 +367,7 @@ kubectl get pvc test-pvc ``` NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -test-pvc Bound pvc-a1b2c3d4-5678-90ab-cdef-1234567890ab 1Gi RWO beta2 10s +test-pvc Bound pvc-a1b2c3d4-5678-90ab-cdef-1234567890ab 1Gi RWO beta3 10s ``` Status should be `Bound`. @@ -399,6 +398,10 @@ kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd status ### Common Issues +**Fewer OSDs than expected (e.g. only ~6 when you have more NVMe drives):** +- Check `deviceFilter`. `"^nvme[01]n1"` only matches `nvme0n1` and `nvme1n1`. Use `"^nvme[0-9]+n1$"` for all `nvmeXn1` data namespaces. +- Confirm with `lsblk -f` which devices should be claimed, then `ceph osd tree` for what Ceph actually took. + **OSDs not starting:** - Verify drives are unformatted: `lsblk -f` - Check deviceFilter matches your drives @@ -424,7 +427,7 @@ Your persistent storage is now ready! - [Provider installation – STEP 9 (TLS)](/docs/providers/setup-and-installation/kubespray/provider-installation-prep#step-9---lets-encrypt-cert-manager-and-tls-secrets) - **Required** for all providers: cert-manager and Gateway TLS - [IP Leases](/docs/providers/setup-and-installation/kubespray/ip-leases) - Enable static IPs -> **Note:** You'll need to configure storage classes in the inventory operator during provider installation to advertise persistent storage capabilities. This is covered in the Provider Installation guide. +> **Note:** You'll need to configure storage classes in the inventory operator during provider installation to advertise persistent storage capabilities (`beta3` for this default NVMe setup). This is covered in the Provider Installation guide. --- diff --git a/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx b/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx index e0e92b2be..88568bd2b 100644 --- a/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx +++ b/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx @@ -285,7 +285,7 @@ attributes: value: "true" ``` -> **Important:** You can only advertise **one storage class** per provider. Choose either beta1 (HDD), beta2 (SSD), or beta3 (NVMe) based on what you configured in Rook-Ceph. +> **Important:** You can only advertise **one storage class** per provider. Match Rook-Ceph: **`beta3` (NVMe) is the default** in the Persistent Storage guide; use `beta2` (SSD) or `beta1` (HDD) only if you configured that instead. --- From 9c5a0c1917f320cbcf6ec8a04a25f506089fee50 Mon Sep 17 00:00:00 2001 From: zblocker64 Date: Fri, 17 Jul 2026 12:16:47 -0500 Subject: [PATCH 4/4] chore: more fixes --- .../Docs/providers/operations/gateway-api-migration/index.md | 3 ++- .../kubespray/provider-installation-prep/index.mdx | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/content/Docs/providers/operations/gateway-api-migration/index.md b/src/content/Docs/providers/operations/gateway-api-migration/index.md index 42145e3a0..db02b44c8 100644 --- a/src/content/Docs/providers/operations/gateway-api-migration/index.md +++ b/src/content/Docs/providers/operations/gateway-api-migration/index.md @@ -56,7 +56,7 @@ kubectl -n akash-gateway describe certificate wildcard-ingress Install the Gateway API CRDs (experimental channel, includes `TCPRoute`) from NGINX Gateway Fabric. These are required before deploying NGINX Gateway Fabric. ```bash -kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v2.5.1" | kubectl apply --server-side -f - +kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v2.6.7" | kubectl apply --server-side -f - ``` Verify the CRDs are installed: @@ -139,6 +139,7 @@ Install NGINX Gateway Fabric via Helm: cd /root/provider helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ + --version 2.6.7 \ --create-namespace \ -n nginx-gateway \ -f values-nginx-gateway-fabric.yaml diff --git a/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx b/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx index 88568bd2b..412acadea 100644 --- a/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx +++ b/src/content/Docs/providers/setup-and-installation/kubespray/provider-installation-prep/index.mdx @@ -497,7 +497,7 @@ Install the Kubernetes Gateway API and NGINX Gateway Fabric (NGF) with **port 44 ### Install Gateway API CRDs ```bash -kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v2.5.1" | kubectl apply --server-side -f - +kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v2.6.7" | kubectl apply --server-side -f - ``` Verify: @@ -562,6 +562,7 @@ Install NGF (from `/root/provider`): cd /root/provider helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ + --version 2.6.7 \ --create-namespace \ -n nginx-gateway \ -f values-nginx-gateway-fabric.yaml