diff --git a/.gitignore b/.gitignore index 0234304a1..07532fa9a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ yarn-error.log* # wrangler local dev .dev.vars .wrangler/ + +# Scratch notes for unpublished posts, never rendered by lib/blog.js +/content/blog/_drafts/ diff --git a/components/MoeExpertRoutingAnimation.jsx b/components/MoeExpertRoutingAnimation.jsx new file mode 100644 index 000000000..2c6424166 --- /dev/null +++ b/components/MoeExpertRoutingAnimation.jsx @@ -0,0 +1,290 @@ +const gpus = [0, 1, 2, 3]; +const PER_GPU = 32; + +export default function MoeExpertRoutingAnimation() { + return ( +
+ +
+

Expert routing animation

+

+ Why a 235B model only does 22B of work per token +

+

+ Every layer of this model has 128 small expert networks, and a tiny router picks just 8 of + them for each token. The other 120 sit still. That is the whole trick of a mixture of + experts: you pay for 235B parameters in memory, but only about 22B of arithmetic per token. +

+ +
+
+ one token arrives + it has already been through attention for this layer +
+ +
router scores all 128 experts, keeps the top 8
+ +
+ {gpus.map((gpu) => { + // 2 of this GPU's 32 experts are picked, so 8 across 4 GPUs + const hot = [3 + gpu, 18 + ((gpu * 5) % 10)]; + return ( +
+

+ GPU {gpu} + experts {gpu * PER_GPU}-{gpu * PER_GPU + PER_GPU - 1} +

+ + ); + })} +
+ +
+
+ In memory + 235B params + + All 128 experts per layer must be resident, which is why the model is big + +
+
+ Active per token + 22B params + + Only the 8 chosen experts do arithmetic, so it runs like a much smaller model + +
+
+ What this costs you + a network hop + + With expert parallelism the token travels to whichever GPU owns its expert, then the + answer travels back + +
+
+
+
+
+ Counts are from the model config: 128 experts per layer, 8 per token, 94 layers. Splitting 128 + experts over 4 GPUs gives 32 each, so on average 2 experts per GPU fire for any given token. + That average is the catch, because routing is not guaranteed to be even. +
+
+ ); +} diff --git a/components/MultiGpuMemoryFitAnimation.jsx b/components/MultiGpuMemoryFitAnimation.jsx new file mode 100644 index 000000000..54032e176 --- /dev/null +++ b/components/MultiGpuMemoryFitAnimation.jsx @@ -0,0 +1,390 @@ +const cases = [ + { + key: 'one', + verdict: 'fail', + title: '1 GPU', + flag: 'will not start', + note: '221 GiB of weights against an 85.51 GiB budget', + parts: [{ label: 'weights', value: '221 GiB', width: '92.08%', color: '#ef4444' }], + log: 'the model is 2.6x larger than the whole budget\nthere is no flag that fixes this', + }, + { + key: 'two', + verdict: 'fail', + title: '2 GPUs', + flag: 'CUDA out of memory', + note: 'about 110 GiB per card, still too much', + parts: [{ label: 'weights per card', value: '110 GiB', width: '46.04%', color: '#f59e0b' }], + log: 'Failed to load model - not enough GPU memory\n95.01 GiB total, of which 438.31 MiB is free', + }, + { + key: 'four', + verdict: 'pass', + title: '4 GPUs', + flag: '621,392 tokens', + note: 'weights fit, with room for about 19 concurrent 32k conversations', + parts: [ + { label: 'weights', value: '55.19 GiB', width: '23.00%', color: '#0098cc' }, + { label: 'KV cache', value: '27.85 GiB', width: '11.60%', color: '#2bb534' }, + ], + log: 'Worker_TP0 Model loading took 55.19 GiB\nAvailable KV cache memory: 27.85 GiB\nGPU KV cache size: 621,392 tokens', + }, +]; + +export default function MultiGpuMemoryFitAnimation() { + return ( +
+ +
+

Memory fit animation

+

+ The same model on 1, 2 and 4 GPUs +

+

+ Every number here came out of a real run. All three bars are drawn to the same scale, and + the dashed line is the 85.51 GiB that vLLM may use on one card at + --gpu-memory-utilization 0.90. A bar reaching past that line means the model does not fit. + Watch it shrink as GPUs are added, and note that it takes 4 before the bar finally lands to + the left of the line. +

+ +
+ {cases.map((c) => ( +
+
+ + {c.title} + {c.note} + + {c.flag} +
+ +
+ what one card must hold + full axis = 240 GiB +
+ +
+ +
+ {c.parts.map((p, i) => ( +
+ + {p.label} {p.value} + +
+ ))} +
+
+ +
{c.log}
+
+ ))} + +
+
+ KV per token, whole model + 188 KiB + 2 x 94 layers x 4 kv heads x 128 head_dim x 2 bytes +
+
+ Per card at TP=4 + 47 KiB + + each card keeps 1 of the 4 kv heads, so the cache divides rather than repeats + +
+
+ Predicted vs reported + 621,337 / 621,392 + + 27.85 GiB divided by 47 KiB, against what vLLM actually printed + +
+
+
+
+
+ Measured on 4x RTX PRO 6000 Blackwell with Qwen3-235B-A22B-Instruct-2507-FP8 on vLLM 0.27.1. + The 1 GPU and 2 GPU bars are what the run actually attempted before failing, not estimates. + Because this model has only 4 key/value heads, its cache is unusually cheap, which is why 4 + cards leave room for about 19 concurrent conversations at the 32,768-token limit we set. +
+
+ ); +} diff --git a/components/MultiGpuSplitModesAnimation.jsx b/components/MultiGpuSplitModesAnimation.jsx new file mode 100644 index 000000000..e58e6c224 --- /dev/null +++ b/components/MultiGpuSplitModesAnimation.jsx @@ -0,0 +1,279 @@ +const modes = [ + { + key: 'tp', + name: 'Tensor parallelism', + flag: '--tensor-parallel-size', + plain: 'Cut every layer into vertical strips. Each GPU holds a strip of all 94 layers.', + talks: 'A lot. Twice per layer, so 188 times per token.', + good: 'Fastest for a single user, because all 4 GPUs work on the same token.', + color: '#0098cc', + }, + { + key: 'pp', + name: 'Pipeline parallelism', + flag: '--pipeline-parallel-size', + plain: 'Cut the stack into horizontal blocks. With 94 layers over 4 GPUs, each one owns about 23 of them.', + talks: 'Barely. One handoff between neighbours per token.', + good: 'Kind to a slow network between GPUs, but a GPU waits its turn.', + color: '#2bb534', + }, + { + key: 'ep', + name: 'Expert parallelism', + flag: '--enable-expert-parallel', + plain: 'Deal the 128 experts out like cards. Each GPU keeps 32 of them, whole.', + talks: 'Medium. Tokens are shipped to whichever GPU owns the expert they need.', + good: 'Only exists for MoE models, and it is how the really big ones are served.', + color: '#a855f7', + }, +]; + +export default function MultiGpuSplitModesAnimation() { + return ( +
+ +
+

Three ways to split animation

+

+ The same model, cut three different ways across four GPUs +

+

+ These are not competing products, they are three different cuts through the same pile of + weights, and you can combine them. Each box below is one GPU. Watch which parts light up, + because that tells you which GPUs are doing work at the same moment. +

+ +
+ {modes.map((mode) => ( +
+

{mode.name}

+ {mode.flag} + + + +
+

+ What it does + {mode.plain} +

+

+ How much it talks + {mode.talks} +

+

+ When it wins + {mode.good} +

+
+
+ ))} +
+
+
+ Layer and expert counts are Qwen3-235B-A22B: 94 layers, 128 experts with 8 picked per token. + Under tensor parallelism all four GPUs light up together on every token. Under pipeline + parallelism they light up in turn, which is the idle time you are trading away. +
+
+ ); +} diff --git a/components/MultiGpuTensorSplitAnimation.jsx b/components/MultiGpuTensorSplitAnimation.jsx new file mode 100644 index 000000000..2e9bbe573 --- /dev/null +++ b/components/MultiGpuTensorSplitAnimation.jsx @@ -0,0 +1,374 @@ +const steps = [ + { label: 'A token arrives', detail: 'all 4 GPUs get the same copy of it' }, + { label: 'Split sideways', detail: 'each GPU owns 16 of the 64 attention heads' }, + { label: 'Work alone', detail: 'no GPU needs to ask the others anything yet' }, + { label: 'Partial answers', detail: 'each GPU has a quarter of the answer' }, + { label: 'Add them up', detail: 'one all-reduce, and all 4 hold the full result' }, +]; + +export default function MultiGpuTensorSplitAnimation() { + return ( +
+ +
+

Tensor parallelism animation

+

+ One layer, sliced four ways +

+

+ This is the part people usually get wrong, so it is worth being precise. The weights get + divided, and the thing flowing through them does not. Every GPU starts each layer holding + an identical copy of the token, does a quarter of the arithmetic on its own slice of the + weights, and ends up with a quarter of an answer. Then they add their quarters together. +

+ +
+
+ the token, 4096 numbers wide + copied to all four GPUs, not divided +
+ +
+ {[0, 1, 2, 3].map((gpu) => ( +
+

+ GPU {gpu} + heads {gpu * 16}-{gpu * 16 + 15} +

+ + ))} +
+ +
+ + +
+ the finished layer output, now identical on all four GPUs + and the next layer does the whole dance again +
+
+ +
+ {steps.map((step, i) => ( +
+ Step {i + 1} + {step.label} + {step.detail} +
+ ))} +
+
+
+ Shapes are Qwen3-235B-A22B: hidden size 4096, 64 attention heads, 4 key/value heads, 94 + layers. Those 4 key/value heads are the reason this model cannot be split cleanly more than 4 + ways, which we come back to later. +
+
+ ); +} diff --git a/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md b/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md new file mode 100644 index 000000000..bbe3dd129 --- /dev/null +++ b/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md @@ -0,0 +1,563 @@ +--- +title: "Running a big LLM across multiple GPUs with vLLM" +seoTitle: "Running a big LLM across multiple GPUs with vLLM" +seoDescription: "A runbook for serving a model too big for one GPU: download to serving in eight steps, with every vLLM flag, startup log line and real error explained, measured on a 235B model across four RTX PRO 6000 cards." +datePublished: 2026-08-18T10:00:00.000Z +slug: running-a-big-llm-across-multiple-gpus-with-vllm +author: shubham-katara +authors: ["shubham-katara", "saiyam-pathak"] +cover: /img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png +tags: ["vllm", "gpu", "nvidia", "llm", "platform-engineering"] +--- + +Sooner or later everyone running models locally hits the same wall. You find a model you want, you look at the download size, and it is bigger than the GPU you own. A 235B model needs roughly 236 GB just for its weights. The card we have holds 96 GB, and even the largest data-centre GPUs available today top out well below 236 GB. So the model does not fit, and no amount of clever flags will make 236 GB squeeze into 96 GB. + +The answer is to use more than one GPU. That part everybody knows. The part that is genuinely confusing is what "use more than one GPU" actually means. Does each GPU get a copy of the model? Does the model get cut in half? Do the GPUs take turns? Which of those is happening, and what does it cost you? + +Let's answer that properly, with a real model on real hardware. + +## What this post covers + +This is the runbook. Eight steps, from downloading a 236 GB model to serving it across four GPUs, with every command, flag, startup log line and real error explained. It is written for the person with root on the box, and it assumes no prior knowledge of distributed computing: if you know what a GPU is and you have run a model locally once, you are qualified. + +It deliberately does not explain the machinery underneath. Why splitting a layer across GPUs makes prefill faster but costs you an all-reduce per layer, why that trade lands differently on decode, and why NVLink is the variable that decides the winner, are all part two, coming next. Where a "why" would otherwise interrupt the work, this post says so and moves on. + +New to the jargon? Every term, flag, and benchmark number here is explained in plain English in the [local LLM glossary](https://blog.kubesimplify.com/local-llm-glossary). + +## The machine and the model + +Numbers mean nothing without the hardware attached, so here it is once. + +**The machine:** a server with 8x NVIDIA RTX PRO 6000 Blackwell Server Edition cards. Each card has 96 GB of memory, and the machine reports 95.01 GiB of that as usable. We borrowed 4 of the 8 cards for this work. + +One detail that matters more than it looks: these GPUs are **not** connected by NVLink. NVLink is NVIDIA's fast direct GPU-to-GPU cable. Without it, GPUs talk to each other over PCIe and through the CPU, which is slower. You can check what you have with one command: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# nvidia-smi topo -m + +| Device | GPU0 | GPU1 | GPU2 | GPU3 | GPU4 | GPU5 | GPU6 | GPU7 | NIC0 | CPU Affinity | NUMA Affinity | GPU NUMA ID | +| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :--- | :---: | :---: | +| **GPU0** | **X** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | SYS | 48-55,176-183 | 6 | N/A | +| **GPU1** | SYS | **X** | SYS | SYS | SYS | SYS | SYS | SYS | PHB | 32-39,160-167 | 4 | N/A | +| **GPU2** | SYS | SYS | **X** | SYS | SYS | SYS | SYS | SYS | SYS | 0-7,128-135 | 0 | N/A | +| **GPU3** | SYS | SYS | SYS | **X** | SYS | SYS | SYS | SYS | SYS | 16-23,144-151 | 2 | N/A | +| **GPU4** | SYS | SYS | SYS | SYS | **X** | SYS | SYS | SYS | SYS | 112-119,240-247 | 14 | N/A | +| **GPU5** | SYS | SYS | SYS | SYS | SYS | **X** | SYS | SYS | SYS | 96-103,224-231 | 12 | N/A | +| **GPU6** | SYS | SYS | SYS | SYS | SYS | SYS | **X** | SYS | SYS | 64-71,192-199 | 8 | N/A | +| **GPU7** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | **X** | SYS | 80-87,208-215 | 10 | N/A | +| **NIC0** | SYS | PHB | SYS | SYS | SYS | SYS | SYS | SYS | **X** | | | | + +**Legend:** + +| Symbol | Description | +| :--- | :--- | +| **X** | Self | +| **SYS** | Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI) | +| **NODE** | Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node | +| **PHB** | Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU) | +| **PXB** | Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge) | +| **PIX** | Connection traversing at most a single PCIe bridge | +| **NV#** | Connection traversing a bonded set of `#` NVLinks | +| **NIC0** | `mlx4_0` | +``` + +On our machine every pair of GPUs reports `SYS`, which means the traffic goes across PCIe and then across the link between the CPU sockets. If you had NVLink you would see `NV1`, `NV2` and so on instead. Keep this in mind, because it changes which splitting method is fastest. + +**The model:** `Qwen/Qwen3-235B-A22B-Instruct-2507-FP8`. Let's unpack that name, because it is doing a lot of work: + +- **235B** is the total parameter count, 235 billion. +- **A22B** means 22 billion **active** parameters. This is a mixture-of-experts model: each layer holds 128 small expert networks and a router picks just 8 of them per token, so you pay for 235B in memory but only about 22B in arithmetic. +- **FP8** is the number format the weights are stored in, 8 bits each, so one byte per parameter. + +**The software:** vLLM 0.27.1 running in the official container, with PyTorch 2.13.0 and CUDA 13.0, on driver 610.43.02. + +--- + +## Step 1: Getting the model onto the machine + +Before anything can be split across GPUs it has to be on disk, and with a model this size that is not a formality. It is the step that bit us hardest, so it goes first. + +### Check your disk first, because this is a real production hazard + +**On a shared machine, filling the disk can take down everything else on it.** This is the part we learned the hard way, and it is worth more than a footnote. Our test box also runs a Kubernetes inference platform. Kubernetes treats free disk as a managed resource called ephemeral-storage, and when free space fell below its eviction threshold, the kubelet did exactly what it is designed to do. + +It evicted pods to reclaim space, tainted the node so nothing new could schedule, and garbage-collected container images. Several of those images had been built locally and existed in no registry, so they could not simply be pulled again. + +Nothing about that is a Kubernetes bug, and nothing about it is specific to our setup. The lesson generalises: **before you download a quarter of a terabyte onto a machine, check what else lives on that disk and what will happen when it fills.** `df -h` before you start, and know your platform's eviction threshold, which is often far higher than "0 bytes free". If the machine is shared, keeping a couple of hundred gigabytes of headroom is not paranoia. + +### The download + +With the headroom confirmed, you download it with the Hugging Face CLI: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# pip install huggingface_hub hf_transfer +root@utho-gpu-rtxpro6000-8-62383:~# HF_XET_HIGH_PERFORMANCE=1 hf download Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 +Downloading bytes: ████████████████████████████████████████████████▏ | 24.4GB, 234MB/s +Reconstructing (incomplete total...): 13%|███████████████▋ | 10.0GB / 80.0GB, 104MB/s +Fetching 34 files: 0%| | 0/34 [00:00 https://blog.kubesimplify.com/ - 2026-08-18T08:33:37.156Z + 2026-08-28T19:13:14.583Z Kubesimplify hello@kubesimplify.com + + Running a big LLM across multiple GPUs with vLLM + + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + 2026-08-18T10:00:00.000Z + 2026-08-18T10:00:00.000Z + A runbook for serving a model too big for one GPU: download to serving in eight steps, with every vLLM flag, startup log line and real error explained, measured on a 235B model across four RTX PRO 6000 cards. + + + + + + The Local LLM Glossary: Every Term, Flag, and Number in Plain English diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cake-layers.png b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cake-layers.png new file mode 100644 index 000000000..ce7555ab3 Binary files /dev/null and b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cake-layers.png differ diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png new file mode 100644 index 000000000..6ed8b13f6 Binary files /dev/null and b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png differ diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg new file mode 100644 index 000000000..9c6a3d465 --- /dev/null +++ b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg @@ -0,0 +1,55 @@ + + + +One big model, four GPUs +how a 235B model is cut up so it fits, and what that costs + +ONE CARD + + + +95 GiB +usable + + + +236 GB of weights +2.3x too big +no flag fixes this + +FOUR CARDS, --tensor-parallel-size 4 + + + +GPU 0 +59 GB +weights +16 of 64 heads + + + +GPU 1 +59 GB +weights +16 of 64 heads + + + +GPU 2 +59 GB +weights +16 of 64 heads + + + +GPU 3 +59 GB +weights +16 of 64 heads + +188 all-reduces per token + +QWEN3-235B-A22B FP8 - 128 EXPERTS, 8 PER TOKEN - vLLM 0.27.1 +tensor, pipeline and expert parallelism explained in plain english +blog.kubesimplify.com + \ No newline at end of file diff --git a/public/llms-full.txt b/public/llms-full.txt index df6842053..871a6c80c 100644 --- a/public/llms-full.txt +++ b/public/llms-full.txt @@ -5,6 +5,566 @@ --- +# Running a big LLM across multiple GPUs with vLLM + +- Canonical: https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm +- Published: 2026-08-18 +- Summary: A runbook for serving a model too big for one GPU: download to serving in eight steps, with every vLLM flag, startup log line and real error explained, measured on a 235B model across four RTX PRO 6000 cards. + +Sooner or later everyone running models locally hits the same wall. You find a model you want, you look at the download size, and it is bigger than the GPU you own. A 235B model needs roughly 236 GB just for its weights. The card we have holds 96 GB, and even the largest data-centre GPUs available today top out well below 236 GB. So the model does not fit, and no amount of clever flags will make 236 GB squeeze into 96 GB. + +The answer is to use more than one GPU. That part everybody knows. The part that is genuinely confusing is what "use more than one GPU" actually means. Does each GPU get a copy of the model? Does the model get cut in half? Do the GPUs take turns? Which of those is happening, and what does it cost you? + +Let's answer that properly, with a real model on real hardware. + +## What this post covers + +This is the runbook. Eight steps, from downloading a 236 GB model to serving it across four GPUs, with every command, flag, startup log line and real error explained. It is written for the person with root on the box, and it assumes no prior knowledge of distributed computing: if you know what a GPU is and you have run a model locally once, you are qualified. + +It deliberately does not explain the machinery underneath. Why splitting a layer across GPUs makes prefill faster but costs you an all-reduce per layer, why that trade lands differently on decode, and why NVLink is the variable that decides the winner, are all part two, coming next. Where a "why" would otherwise interrupt the work, this post says so and moves on. + +New to the jargon? Every term, flag, and benchmark number here is explained in plain English in the [local LLM glossary](https://blog.kubesimplify.com/local-llm-glossary). + +## The machine and the model + +Numbers mean nothing without the hardware attached, so here it is once. + +**The machine:** a server with 8x NVIDIA RTX PRO 6000 Blackwell Server Edition cards. Each card has 96 GB of memory, and the machine reports 95.01 GiB of that as usable. We borrowed 4 of the 8 cards for this work. + +One detail that matters more than it looks: these GPUs are **not** connected by NVLink. NVLink is NVIDIA's fast direct GPU-to-GPU cable. Without it, GPUs talk to each other over PCIe and through the CPU, which is slower. You can check what you have with one command: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# nvidia-smi topo -m + +| Device | GPU0 | GPU1 | GPU2 | GPU3 | GPU4 | GPU5 | GPU6 | GPU7 | NIC0 | CPU Affinity | NUMA Affinity | GPU NUMA ID | +| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :--- | :---: | :---: | +| **GPU0** | **X** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | SYS | 48-55,176-183 | 6 | N/A | +| **GPU1** | SYS | **X** | SYS | SYS | SYS | SYS | SYS | SYS | PHB | 32-39,160-167 | 4 | N/A | +| **GPU2** | SYS | SYS | **X** | SYS | SYS | SYS | SYS | SYS | SYS | 0-7,128-135 | 0 | N/A | +| **GPU3** | SYS | SYS | SYS | **X** | SYS | SYS | SYS | SYS | SYS | 16-23,144-151 | 2 | N/A | +| **GPU4** | SYS | SYS | SYS | SYS | **X** | SYS | SYS | SYS | SYS | 112-119,240-247 | 14 | N/A | +| **GPU5** | SYS | SYS | SYS | SYS | SYS | **X** | SYS | SYS | SYS | 96-103,224-231 | 12 | N/A | +| **GPU6** | SYS | SYS | SYS | SYS | SYS | SYS | **X** | SYS | SYS | 64-71,192-199 | 8 | N/A | +| **GPU7** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | **X** | SYS | 80-87,208-215 | 10 | N/A | +| **NIC0** | SYS | PHB | SYS | SYS | SYS | SYS | SYS | SYS | **X** | | | | + +**Legend:** + +| Symbol | Description | +| :--- | :--- | +| **X** | Self | +| **SYS** | Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI) | +| **NODE** | Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node | +| **PHB** | Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU) | +| **PXB** | Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge) | +| **PIX** | Connection traversing at most a single PCIe bridge | +| **NV#** | Connection traversing a bonded set of `#` NVLinks | +| **NIC0** | `mlx4_0` | +``` + +On our machine every pair of GPUs reports `SYS`, which means the traffic goes across PCIe and then across the link between the CPU sockets. If you had NVLink you would see `NV1`, `NV2` and so on instead. Keep this in mind, because it changes which splitting method is fastest. + +**The model:** `Qwen/Qwen3-235B-A22B-Instruct-2507-FP8`. Let's unpack that name, because it is doing a lot of work: + +- **235B** is the total parameter count, 235 billion. +- **A22B** means 22 billion **active** parameters. This is a mixture-of-experts model: each layer holds 128 small expert networks and a router picks just 8 of them per token, so you pay for 235B in memory but only about 22B in arithmetic. +- **FP8** is the number format the weights are stored in, 8 bits each, so one byte per parameter. + +**The software:** vLLM 0.27.1 running in the official container, with PyTorch 2.13.0 and CUDA 13.0, on driver 610.43.02. + +--- + +## Step 1: Getting the model onto the machine + +Before anything can be split across GPUs it has to be on disk, and with a model this size that is not a formality. It is the step that bit us hardest, so it goes first. + +### Check your disk first, because this is a real production hazard + +**On a shared machine, filling the disk can take down everything else on it.** This is the part we learned the hard way, and it is worth more than a footnote. Our test box also runs a Kubernetes inference platform. Kubernetes treats free disk as a managed resource called ephemeral-storage, and when free space fell below its eviction threshold, the kubelet did exactly what it is designed to do. + +It evicted pods to reclaim space, tainted the node so nothing new could schedule, and garbage-collected container images. Several of those images had been built locally and existed in no registry, so they could not simply be pulled again. + +Nothing about that is a Kubernetes bug, and nothing about it is specific to our setup. The lesson generalises: **before you download a quarter of a terabyte onto a machine, check what else lives on that disk and what will happen when it fills.** `df -h` before you start, and know your platform's eviction threshold, which is often far higher than "0 bytes free". If the machine is shared, keeping a couple of hundred gigabytes of headroom is not paranoia. + +### The download + +With the headroom confirmed, you download it with the Hugging Face CLI: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# pip install huggingface_hub hf_transfer +root@utho-gpu-rtxpro6000-8-62383:~# HF_XET_HIGH_PERFORMANCE=1 hf download Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 +Downloading bytes: ████████████████████████████████████████████████▏ | 24.4GB, 234MB/s +Reconstructing (incomplete total...): 13%|███████████████▋ | 10.0GB / 80.0GB, 104MB/s +Fetching 34 files: 0%| | 0/34 [00:00 Deep dives on Kubernetes, AI infrastructure, GitOps, and the cloud-native stack, written by practitioners. en-us - Tue, 18 Aug 2026 09:00:00 GMT + Tue, 18 Aug 2026 10:00:00 GMT Kubesimplify static blog + + Running a big LLM across multiple GPUs with vLLM + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + Tue, 18 Aug 2026 10:00:00 GMT + A runbook for serving a model too big for one GPU: download to serving in eight steps, with every vLLM flag, startup log line and real error explained, measured on a 235B model across four RTX PRO 6000 cards. + vllmgpunvidiallmplatform-engineering + The Local LLM Glossary: Every Term, Flag, and Number in Plain English https://blog.kubesimplify.com/local-llm-glossary diff --git a/scripts/gen-local-llm-glossary-cover.mjs b/scripts/gen-local-llm-glossary-cover.mjs index 803b42205..45c0ec60a 100644 --- a/scripts/gen-local-llm-glossary-cover.mjs +++ b/scripts/gen-local-llm-glossary-cover.mjs @@ -1,5 +1,5 @@ // Excalidraw-style cover for the local LLM glossary post. -// Sketch helpers shared with scripts/gen-two-gpu-vllm-cover.mjs. +// Sketch helpers shared with scripts/gen-multi-gpu-vllm-cover.mjs. import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; diff --git a/scripts/gen-multi-gpu-vllm-cover.mjs b/scripts/gen-multi-gpu-vllm-cover.mjs new file mode 100644 index 000000000..6c1be76af --- /dev/null +++ b/scripts/gen-multi-gpu-vllm-cover.mjs @@ -0,0 +1,228 @@ +// Excalidraw-style cover for the multi-GPU vLLM article. +// Sketch helpers shared with scripts/gen-hami-diagrams.mjs. +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +let seed = 42; +const random = () => { + seed = (seed * 16807) % 2147483647; + return seed / 2147483647; +}; +const jitter = (amount) => (random() - 0.5) * amount * 2; + +const COLORS = { + ink: '#172033', + muted: '#5c677d', + green: { stroke: '#5d8f00', fill: '#d8f5a2' }, + blue: { stroke: '#1971c2', fill: '#a5d8ff' }, + violet: { stroke: '#862e9c', fill: '#eebefa' }, + orange: { stroke: '#d9480f', fill: '#ffd8a8' }, + red: { stroke: '#c92a2a', fill: '#ffc9c9' }, + teal: { stroke: '#087f5b', fill: '#b2f2bb' }, + gray: { stroke: '#495057', fill: '#e9ecef' }, +}; + +const FONT = 'Chalkboard SE, Comic Sans MS, sans-serif'; + +function roughLine(x1, y1, x2, y2, amount = 1.8) { + const middleX = (x1 + x2) / 2 + jitter(amount * 1.5); + const middleY = (y1 + y2) / 2 + jitter(amount * 1.5); + return `M ${(x1 + jitter(amount)).toFixed(1)} ${(y1 + jitter(amount)).toFixed(1)} Q ${middleX.toFixed(1)} ${middleY.toFixed(1)} ${(x2 + jitter(amount)).toFixed(1)} ${(y2 + jitter(amount)).toFixed(1)}`; +} + +class Sketch { + constructor(width, height, background = '#ffffff') { + this.width = width; + this.height = height; + this.background = background; + this.parts = []; + this.defs = []; + this.clipId = 0; + } + + add(value) { + this.parts.push(value); + } + + rect(x, y, width, height, options = {}) { + const { + stroke = COLORS.ink, + fill, + strokeWidth = 2.4, + dashed = false, + hachure = true, + radius = 7, + } = options; + + if (fill) { + if (hachure) { + // Hatch lines are clipped in math rather than with an SVG clipPath so + // the file renders identically in renderers without clipPath support. + const hatch = []; + for (let offset = -height; offset < width; offset += 11) { + const tMin = Math.max(0, -offset / height); + const tMax = Math.min(1, (width - offset) / height); + if (tMax - tMin < 0.05) continue; + const x1 = x + offset + height * tMin; + const y1 = y + height - height * tMin; + const x2 = x + offset + height * tMax; + const y2 = y + height - height * tMax; + hatch.push(roughLine(x1, y1, x2, y2, 1)); + } + this.add(``); + } else { + this.add(``); + } + } + + const points = [[x, y], [x + width, y], [x + width, y + height], [x, y + height]]; + for (let pass = 0; pass < 2; pass += 1) { + const path = points.map((point, index) => { + const next = points[(index + 1) % points.length]; + return roughLine(point[0], point[1], next[0], next[1], pass === 0 ? 2 : 1.2); + }).join(' '); + this.add(``); + } + } + + line(x1, y1, x2, y2, options = {}) { + const { stroke = COLORS.ink, strokeWidth = 2.4, dashed = false } = options; + this.add(``); + } + + arrow(x1, y1, x2, y2, options = {}) { + const { stroke = COLORS.ink, strokeWidth = 2.6, dashed = false } = options; + this.line(x1, y1, x2, y2, { stroke, strokeWidth, dashed }); + const angle = Math.atan2(y2 - y1, x2 - x1); + const length = 14; + for (const offset of [Math.PI * 0.82, -Math.PI * 0.82]) { + this.line( + x2, + y2, + x2 + length * Math.cos(angle + offset), + y2 + length * Math.sin(angle + offset), + { stroke, strokeWidth } + ); + } + } + + text(x, y, value, options = {}) { + const { + size = 22, + color = COLORS.ink, + anchor = 'middle', + weight = 500, + family = FONT, + } = options; + const safe = String(value) + .replace(/&/g, '&') + .replace(//g, '>'); + this.add(`${safe}`); + } + + lines(x, y, values, options = {}) { + const lineHeight = (options.size || 22) * (options.lineHeight || 1.28); + values.forEach((value, index) => this.text(x, y + index * lineHeight, value, options)); + } + + save(path) { + const svg = ` +${this.defs.join('')} + +${this.parts.join('\n')} +`; + writeFileSync(path, svg); + } +} + +const output = process.argv[2] || '.'; +mkdirSync(output, { recursive: true }); + + +const W = 1200; +const H = 630; +const sketch = new Sketch(W, H, '#fdfdfb'); + +sketch.text(64, 82, 'One big model, four GPUs', { size: 50, weight: 800, anchor: 'start' }); +sketch.text(64, 119, 'how a 235B model is cut up so it fits, and what that costs', { + size: 22, + color: COLORS.muted, + anchor: 'start', +}); +sketch.line(64, 139, 760, 139, { stroke: COLORS.muted, strokeWidth: 1.6, dashed: true }); + +// ── left: the model does not fit on one card ────────────── +sketch.text(64, 186, 'ONE CARD', { size: 18, weight: 800, anchor: 'start', color: COLORS.red.stroke }); + +const bY = 206; +sketch.rect(64, bY, 210, 132, { stroke: COLORS.gray.stroke, fill: '#ffffff', hachure: false, dashed: true }); +sketch.text(169, bY + 30, '95 GiB', { size: 19, color: COLORS.muted }); +sketch.text(169, bY + 54, 'usable', { size: 15, color: COLORS.muted }); + +// overflowing weights bar +sketch.rect(78, bY + 72, 330, 46, { stroke: COLORS.red.stroke, fill: COLORS.red.fill }); +sketch.text(200, bY + 95, '236 GB of weights', { size: 19, weight: 800, color: COLORS.red.stroke }); + +sketch.text(64, bY + 164, '2.3x too big', { size: 26, weight: 800, anchor: 'start', color: COLORS.red.stroke }); +sketch.text(64, bY + 192, 'no flag fixes this', { size: 16, anchor: 'start', color: COLORS.muted }); + +// ── divider ─────────────────────────────────────────────── +sketch.line(452, 186, 452, 452, { stroke: COLORS.muted, strokeWidth: 1.6, dashed: true }); + +// ── right: four cards, each holds a quarter ─────────────── +sketch.text(516, 186, 'FOUR CARDS, --tensor-parallel-size 4', { + size: 18, + weight: 800, + anchor: 'start', + color: COLORS.teal.stroke, +}); + +const cw = 145; +const gap = 10; +const gY = 206; +const palette = [COLORS.blue, COLORS.green, COLORS.violet, COLORS.orange]; +[0, 1, 2, 3].forEach((gpu) => { + const x = 516 + gpu * (cw + gap); + const c = palette[gpu]; + sketch.rect(x, gY, cw, 132, { stroke: c.stroke, fill: c.fill }); + sketch.text(x + cw / 2, gY + 30, `GPU ${gpu}`, { size: 20, weight: 800, color: c.stroke }); + sketch.text(x + cw / 2, gY + 60, '59 GB', { size: 18, weight: 700 }); + sketch.text(x + cw / 2, gY + 84, 'weights', { size: 14, color: COLORS.muted }); + sketch.text(x + cw / 2, gY + 112, '16 of 64 heads', { size: 13, color: COLORS.muted }); +}); + +// all-reduce arrows under the row of cards +const arrowY = gY + 154; +sketch.line(516 + 40, arrowY, 516 + 3 * (cw + gap) + cw - 40, arrowY, { + stroke: COLORS.violet.stroke, + dashed: true, +}); +sketch.text(516 + (3 * (cw + gap) + cw) / 2, arrowY + 30, '188 all-reduces per token', { + size: 18, + weight: 800, + color: COLORS.violet.stroke, +}); + +// ── footer ──────────────────────────────────────────────── +sketch.line(64, 516, W - 64, 516, { stroke: COLORS.muted, strokeWidth: 1.6 }); +sketch.text(64, 552, 'QWEN3-235B-A22B FP8 - 128 EXPERTS, 8 PER TOKEN - vLLM 0.27.1', { + size: 18, + weight: 800, + anchor: 'start', + color: COLORS.ink, +}); +sketch.text(64, 582, 'tensor, pipeline and expert parallelism explained in plain english', { + size: 16, + anchor: 'start', + color: COLORS.muted, +}); +sketch.text(W - 64, 582, 'blog.kubesimplify.com', { + size: 16, + weight: 700, + anchor: 'end', + color: COLORS.muted, +}); + +sketch.save(join(output, 'cover.svg')); +console.log(`Wrote multi-GPU vLLM cover to ${output}`); diff --git a/vercel.json b/vercel.json index eaa7bd155..2e0044f80 100644 --- a/vercel.json +++ b/vercel.json @@ -906,6 +906,11 @@ "destination": "https://blog.kubesimplify.com/ready-for-wasm-day-2023", "permanent": true }, + { + "source": "/blog/running-a-big-llm-across-multiple-gpus-with-vllm", + "destination": "https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm", + "permanent": true + }, { "source": "/blog/sharing-gpus-in-kubernetes-with-hami", "destination": "https://blog.kubesimplify.com/sharing-gpus-in-kubernetes-with-hami", @@ -2916,6 +2921,17 @@ } ] }, + { + "source": "/running-a-big-llm-across-multiple-gpus-with-vllm", + "destination": "https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm", + "permanent": true, + "has": [ + { + "type": "host", + "value": "kubesimplify.com" + } + ] + }, { "source": "/sharing-gpus-in-kubernetes-with-hami", "destination": "https://blog.kubesimplify.com/sharing-gpus-in-kubernetes-with-hami",