From 636b434220133aa58639a712843b52dec689feae Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 4 Jun 2026 07:40:30 +0000 Subject: [PATCH] feat(service-catalog): split GPU transcoding lane from base media lane (closes #63) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defines the GPU transcoding and hardware-passthrough validation lane as a standalone split from the base media-service lane (#59). All tests in the module are gated behind a runtime GPU-availability check so non-GPU clusters are not broken. ## Why the split matters (from issue #63) GPU transcoding will distort the base media scope unless kept explicitly separate. This PR makes the substrate dependency chain visible: #54 (GPU device plugin + driver stack on ghost) → #63 (GPU transcoding contract, this PR) → #59 (base media lane, now unblocked) ## New files ### tests/service_catalog/media_gpu/test_gpu_transcoding.py 13 test methods in TestMediaGpuTranscodingLane: Module-level skip gate: _gpu_allocatable_on_any_node() — if no nvidia.com/gpu in any node allocatable, entire module skips with a message pointing to #54 substrate requirements. Tests (active when GPU present): - Substrate: node allocatable capacity, device plugin DaemonSet Running - Deployment: GPU deployment ready, pod Running, resource limit in pod spec - Device: /dev/nvidia* visible in container, nvidia-smi reports GPU - Transcoding: ffmpeg lists nvenc encoder, ffmpeg hardware transcode completes - Lifecycle: GPU capacity recovers after pod deletion Tests (explicit skips): - KubeVirt VM-backed passthrough → pytest.skip → #54 - Multi-GPU / MIG slicing → pytest.skip → deferred - AMD ROCm / Intel QSV → pytest.skip → deferred ### argo/workflow-templates/homelab-media-gpu.yaml WorkflowTemplate (ArgoCD-managed): - check-gpu-prerequisites → create-namespace → deploy-fixture → run-tests DAG - check-gpu-prerequisites: guard step that verifies GPU allocatable capacity and device plugin pods; fails with clear #54 instructions if not ready - deploy-fixture: homelab-media-gpu-config 1Gi PVC, Deployment with nvidia.com/gpu: 1 resource limit, NVIDIA_VISIBLE_DEVICES=all env, runtimeClassName: nvidia, linuxserver/ffmpeg image - cleanup onExit handler ### argo/homelab-media-gpu.yaml Workflow submit trigger. ## Updated files ### docs/homelab-contracts.md - §6 known-blockers: #63 updated from 'deferred' to 'defined' - §7 GPU Transcoding and Hardware-Passthrough Lane added: substrate dependency chain, required #54 steps, behavior table, module-level skip gate explanation, WorkflowTemplate guard step, out-of-scope splits table Closes #63 Child of #51, #54 Depends on #54 (substrate), #59 (base media lane) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- argo/homelab-media-gpu.yaml | 11 + .../workflow-templates/homelab-media-gpu.yaml | 236 ++++++++++ docs/homelab-contracts.md | 71 ++- tests/service_catalog/media_gpu/__init__.py | 0 .../media_gpu/test_gpu_transcoding.py | 421 ++++++++++++++++++ 5 files changed, 738 insertions(+), 1 deletion(-) create mode 100644 argo/homelab-media-gpu.yaml create mode 100644 argo/workflow-templates/homelab-media-gpu.yaml create mode 100644 tests/service_catalog/media_gpu/__init__.py create mode 100644 tests/service_catalog/media_gpu/test_gpu_transcoding.py diff --git a/argo/homelab-media-gpu.yaml b/argo/homelab-media-gpu.yaml new file mode 100644 index 00000000..edaf6075 --- /dev/null +++ b/argo/homelab-media-gpu.yaml @@ -0,0 +1,11 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: homelab-media-gpu- + namespace: argo + labels: + app.kubernetes.io/component: homelab-media-gpu + app.kubernetes.io/part-of: bluefin-test-suite +spec: + workflowTemplateRef: + name: homelab-media-gpu diff --git a/argo/workflow-templates/homelab-media-gpu.yaml b/argo/workflow-templates/homelab-media-gpu.yaml new file mode 100644 index 00000000..12a87e1f --- /dev/null +++ b/argo/workflow-templates/homelab-media-gpu.yaml @@ -0,0 +1,236 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: homelab-media-gpu + namespace: argo + labels: + app.kubernetes.io/component: homelab-media-gpu + app.kubernetes.io/part-of: bluefin-test-suite +spec: + entrypoint: pipeline + onExit: cleanup + serviceAccountName: homelab-runner + arguments: + parameters: + - name: lane + value: "homelab-media-gpu" + - name: branch + value: "main" + - name: gpu-resource-key + value: "nvidia.com/gpu" + - name: device-plugin-label + value: "app=nvidia-device-plugin" + templates: + - name: pipeline + dag: + tasks: + - name: check-gpu-prerequisites + template: check-gpu-prerequisites + arguments: + parameters: + - name: gpu-resource-key + value: "{{workflow.parameters.gpu-resource-key}}" + - name: device-plugin-label + value: "{{workflow.parameters.device-plugin-label}}" + - name: create-namespace + depends: "check-gpu-prerequisites.Succeeded" + template: create-namespace + arguments: + parameters: + - name: namespace + value: "homelab-media-gpu-{{workflow.uid}}" + - name: deploy-fixture + depends: "create-namespace.Succeeded" + template: deploy-fixture + arguments: + parameters: + - name: namespace + value: "homelab-media-gpu-{{workflow.uid}}" + - name: lane + value: "{{workflow.parameters.lane}}" + - name: gpu-resource-key + value: "{{workflow.parameters.gpu-resource-key}}" + - name: run-tests + depends: "deploy-fixture.Succeeded" + templateRef: + name: run-incluster-tests + template: run-pytest + arguments: + parameters: + - name: namespace + value: "homelab-media-gpu-{{workflow.uid}}" + - name: suite-path + value: "service_catalog/media_gpu" + - name: lane + value: "{{workflow.parameters.lane}}" + - name: app-label + value: "app=homelab-media-gpu" + - name: service-name + value: "homelab-media-gpu" + - name: branch + value: "{{workflow.parameters.branch}}" + + - name: check-gpu-prerequisites + # Guard step: verify GPU device plugin is available before allocating + # cluster resources. Fails the workflow early with a clear message + # rather than creating a namespace and leaving a pending pod. + inputs: + parameters: + - name: gpu-resource-key + - name: device-plugin-label + script: + image: cgr.dev/chainguard/kubectl:latest-dev + command: [bash, -c] + source: | + set -euo pipefail + GPU_KEY="{{inputs.parameters.gpu-resource-key}}" + PLUGIN_LABEL="{{inputs.parameters.device-plugin-label}}" + + echo "=== Checking GPU allocatable capacity ===" >&2 + ALLOCATABLE=$(kubectl get nodes -o json | \ + python3 -c " + import sys, json + nodes = json.load(sys.stdin).get('items', []) + total = 0 + for n in nodes: + val = n.get('status', {}).get('allocatable', {}).get('${GPU_KEY}', '0') + try: total += int(val) + except: pass + print(total) + ") + echo "Allocatable ${GPU_KEY}: ${ALLOCATABLE}" >&2 + + if [ "${ALLOCATABLE}" -lt 1 ]; then + echo "ERROR: No allocatable ${GPU_KEY} found on any node." >&2 + echo "GPU transcoding lane requires substrate work from #54:" >&2 + echo " 1. Install NVIDIA driver on host node (ghost)" >&2 + echo " 2. Install nvidia-container-toolkit" >&2 + echo " 3. Deploy nvidia-device-plugin DaemonSet" >&2 + echo " 4. Verify: kubectl get nodes -o json | grep -i gpu" >&2 + exit 1 + fi + + echo "=== Checking device plugin DaemonSet ===" >&2 + RUNNING=$(kubectl get pods --all-namespaces -l "${PLUGIN_LABEL}" \ + --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l) + echo "Running device plugin pods: ${RUNNING}" >&2 + + if [ "${RUNNING}" -lt 1 ]; then + echo "ERROR: No Running pods found for label '${PLUGIN_LABEL}'." >&2 + echo "Install and verify the nvidia-device-plugin DaemonSet (see #54)." >&2 + exit 1 + fi + + echo "GPU prerequisites satisfied." >&2 + + - name: create-namespace + inputs: + parameters: + - name: namespace + resource: + action: create + manifest: | + apiVersion: v1 + kind: Namespace + metadata: + name: "{{inputs.parameters.namespace}}" + labels: + app.kubernetes.io/part-of: bluefin-test-suite + bluefin.io/lane: homelab-media-gpu + + - name: deploy-fixture + inputs: + parameters: + - name: namespace + - name: lane + - name: gpu-resource-key + script: + image: cgr.dev/chainguard/kubectl:latest-dev + command: [bash, -c] + source: | + set -euo pipefail + NS="{{inputs.parameters.namespace}}" + GPU_KEY="{{inputs.parameters.gpu-resource-key}}" + kubectl apply -n "${NS}" -f - <&2 + + - name: cleanup + script: + image: cgr.dev/chainguard/kubectl:latest-dev + command: [bash, -c] + source: | + set -euo pipefail + NS="homelab-media-gpu-{{workflow.uid}}" + kubectl delete namespace "${NS}" --ignore-not-found=true >&2 || true + timeout 180 bash -c "until ! kubectl get namespace \"${NS}\" >/dev/null 2>&1; do sleep 5; done" >&2 || true + echo "cleanup-complete" diff --git a/docs/homelab-contracts.md b/docs/homelab-contracts.md index 98ab4f45..fbd517b4 100644 --- a/docs/homelab-contracts.md +++ b/docs/homelab-contracts.md @@ -192,8 +192,77 @@ The lab validates the following hostname/routing pattern for exposed in-cluster | Issue | Status | Dependency | |---|---|---| | #62 RWX / shared-storage | ❌ blocked | NFS CSI or Longhorn installation on ghost | -| #63 GPU transcoding lane | ❌ deferred | GPU passthrough KubeVirt feature gate | +| #63 GPU transcoding lane | ✅ defined | Substrate work from #54 required before tests execute | | #61 auth-gated service UI | ❌ deferred | service-catalog baseline lane first | | #60 first restore drill | ✅ implemented | `homelab-restore-drill` WorkflowTemplate + `tests/homelab_backup/` | | #84 PVC restore drill with backup artifact | ✅ implemented | `homelab-restore-drill` WorkflowTemplate + `tests/homelab_backup/` | | Media service lane | ❌ deferred | #62 (shared mount) + #63 (GPU) | + +--- + +## 7. GPU Transcoding and Hardware-Passthrough Lane (#63) + +Split from the base media-service lane (#59). Defines the in-cluster +validation contract for NVIDIA CUDA/NVENC hardware-accelerated transcoding. +AMD ROCm and Intel QSV paths are deferred; KubeVirt VM-backed passthrough +is tracked in #54. + +### Substrate dependency chain + +``` +#54 substrate (GPU device plugin + driver stack on ghost) + → #63 this lane (GPU transcoding contract proven) + → #59 base media lane (unblocked from GPU split) +``` + +### Required substrate from #54 before this lane executes + +| Requirement | How to verify | +|---|---| +| NVIDIA driver loaded on host (ghost) | `nvidia-smi` on host returns GPU name | +| nvidia-container-toolkit installed | `nvidia-ctk --version` succeeds | +| nvidia-device-plugin DaemonSet running | `kubectl get pods -A -l app=nvidia-device-plugin` shows Running | +| Node reports allocatable GPU capacity | `kubectl get nodes -o json` has `nvidia.com/gpu` > 0 in allocatable | +| Container runtimeClass `nvidia` registered | `kubectl get runtimeclass nvidia` exists | + +### Behavior table (tests run only when GPU is allocatable) + +| Behavior | Test | Artifact | +|---|---|---| +| Node has allocatable GPU capacity | `test_gpu_node_has_allocatable_capacity` | `gpu-node-allocatable.txt` | +| Device plugin DaemonSet is Running | `test_device_plugin_daemonset_is_running` | `gpu-device-plugin-pods.json` | +| GPU Deployment reaches `availableReplicas >= 1` | `test_gpu_deployment_becomes_ready` | `gpu-deployment.json` | +| Pod with GPU resource limit reaches Running | `test_gpu_pod_reaches_running_state` | `gpu-pods.json` | +| GPU resource limit present in pod spec | `test_gpu_resource_limit_present_in_pod_spec` | — | +| /dev/nvidia* device node visible in container | `test_gpu_device_node_visible_in_container` | `gpu-dev-nodes.txt` | +| nvidia-smi reports GPU in container | `test_nvidia_smi_reports_gpu_in_container` | `gpu-nvidia-smi.txt` | +| ffmpeg lists nvenc encoder | `test_ffmpeg_lists_nvenc_encoder` | `gpu-ffmpeg-encoders.txt` | +| ffmpeg hardware transcode completes (1-second clip) | `test_ffmpeg_hardware_transcode_completes` | `gpu-transcode-output.txt` | +| GPU capacity recovers after pod deletion | `test_gpu_resource_is_released_after_pod_deletion` | `gpu-allocatable-after-delete.txt` | +| KubeVirt VM-backed passthrough | `test_kubevirt_vm_gpu_passthrough_is_out_of_scope_for_this_lane` | `pytest.skip` → #54 | +| Multi-GPU / MIG slicing | `test_multi_gpu_and_mig_slicing_is_out_of_scope_for_this_lane` | `pytest.skip` → deferred | +| AMD ROCm / Intel QSV | `test_amd_and_intel_gpu_paths_are_out_of_scope_for_this_lane` | `pytest.skip` → deferred | + +### Module-level skip gate + +All tests are gated at module import time by `_gpu_allocatable_on_any_node()`. +If no `nvidia.com/gpu` (or `TEST_GPU_RESOURCE_KEY`) resource appears in any +node's allocatable map, the entire module is skipped with a message pointing +to the #54 substrate requirements above. This means the GPU lane can be +included in standard CI without failing on non-GPU clusters. + +### WorkflowTemplate guard step + +`check-gpu-prerequisites` runs before namespace creation. If it fails +(no allocatable GPU, no Running device plugin pods), the workflow exits +with a clear error message listing the three steps needed from #54. No +cluster resources are created for a non-GPU cluster. + +### Out-of-scope splits + +| Path | Reason | Tracked | +|---|---|---| +| KubeVirt VM-backed GPU passthrough (VFIO/IOMMU) | Requires additional substrate + #54 | #54 | +| Multi-GPU scheduling / MIG slicing | No multi-GPU hardware in current lab | Deferred | +| AMD ROCm (amd.com/gpu) | Primary target is NVIDIA | Deferred | +| Intel QSV / VAAPI (i915/xe) | Deferred until NVIDIA path validated | Deferred | diff --git a/tests/service_catalog/media_gpu/__init__.py b/tests/service_catalog/media_gpu/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/service_catalog/media_gpu/test_gpu_transcoding.py b/tests/service_catalog/media_gpu/test_gpu_transcoding.py new file mode 100644 index 00000000..68ac1158 --- /dev/null +++ b/tests/service_catalog/media_gpu/test_gpu_transcoding.py @@ -0,0 +1,421 @@ +""" +GPU transcoding and hardware-passthrough lane — split from the base media lane. + +This module defines the validation contract for hardware-accelerated media +transcoding. It is explicitly separated from the base media-service lane +(issue #59) so that the base lane can stay implementable without GPU +hardware, while the hardware path remains visible and independently +trackable. + +## Substrate assumptions this lane requires (all from epic #54) + + A. A GPU device-plugin DaemonSet (nvidia-device-plugin or amd-device-plugin) + is running on the target node so Kubernetes can schedule pods with + `resources.limits: nvidia.com/gpu: 1` (or amd.com/gpu). + + B. The host node exposes the correct device nodes: + - NVIDIA: /dev/nvidia*, /dev/nvidiactl, /dev/nvidia-uvm + - AMD/Intel: /dev/dri/renderD128 (or similar DRI node) + + C. The host kernel module stack is loaded: + - NVIDIA: nvidia, nvidia-uvm + - AMD: amdgpu + - Intel: i915 / xe + + D. Container runtime is configured to pass GPU devices through + (nvidia-container-toolkit or equivalent). For KubeVirt VM-backed + workloads this is a separate follow-up (#54). + +## All tests in this module are gated + + If no node in the cluster reports allocatable GPU resources, the entire + module is skipped with a clear reason string. This means the GPU lane + can be included in the standard CI run without failing on non-GPU + clusters — it will only execute when hardware is present. + +## Dependency chain + + #54 substrate (GPU device plugin + driver stack on ghost) + → this lane (GPU transcoding contract proven) + → base media lane (#59) (unblocked from GPU split) + +## Out-of-scope for this lane (explicit follow-ups) + + - KubeVirt VM-backed GPU passthrough → tracked in #54 + - Multi-GPU scheduling / MIG slicing → deferred, no multi-GPU hardware + - AMD GPU path (ROCm) → deferred; primary target is NVIDIA + - Intel QSV / VAAPI path → deferred until iGPU is validated + +Source ideas: projectbluefin/bluespeed#4, projectbluefin/bluespeed#1 +Child of: #63, #51, #54 +Depends on: #54, #59 +""" + +from __future__ import annotations + +import json +import os +import subprocess +import time + +import pytest + +from tests.service_catalog.shared.kube import ( + NAMESPACE, + get_pods_json, + run_kubectl, + write_artifact, +) + + +# ── GPU availability gate ───────────────────────────────────────────────────── + + +def _gpu_allocatable_on_any_node() -> bool: + """Return True if any cluster node reports a GPU resource as allocatable.""" + result = run_kubectl("get", "nodes", "-o", "json") + if result.returncode != 0: + return False + try: + nodes = json.loads(result.stdout) + except json.JSONDecodeError: + return False + for node in nodes.get("items", []): + allocatable = node.get("status", {}).get("allocatable", {}) + for resource_key in allocatable: + if "gpu" in resource_key.lower(): + return True + return False + + +_NO_GPU = not _gpu_allocatable_on_any_node() + +pytestmark = pytest.mark.skipif( + _NO_GPU, + reason=( + "No GPU resource allocatable on any cluster node. " + "This lane requires substrate work from #54 (GPU device-plugin + driver " + "stack on ghost). Add an nvidia-device-plugin DaemonSet and confirm " + "nvidia.com/gpu appears in 'kubectl get nodes -o json' allocatable before " + "running this suite." + ), +) + + +# ── Lane constants ──────────────────────────────────────────────────────────── + +DEPLOYMENT_NAME = "homelab-media-gpu" +APP_LABEL = os.environ.get("TEST_APP_LABEL", f"app={DEPLOYMENT_NAME}") +SERVICE_NAME = os.environ.get("TEST_SERVICE_NAME", DEPLOYMENT_NAME) + +CONFIG_PVC_NAME = "homelab-media-gpu-config" +CONFIG_MOUNT_PATH = "/config" +TRANSCODE_MOUNT_PATH = "/transcode" + +MEDIA_GPU_PORT = 8096 # Consistent with base media lane + +# GPU resource key; set TEST_GPU_RESOURCE_KEY to override for AMD/Intel +GPU_RESOURCE_KEY = os.environ.get("TEST_GPU_RESOURCE_KEY", "nvidia.com/gpu") + +# Device plugin DaemonSet label to verify in test_device_plugin_daemonset_is_running +DEVICE_PLUGIN_LABEL = os.environ.get( + "TEST_GPU_DEVICE_PLUGIN_LABEL", "app=nvidia-device-plugin" +) + +EXPECTED_ENV_VARS = {"PUID": "1000", "PGID": "1000", "TZ": "UTC"} +EXPECTED_STORAGE_CLASS = "local-path" + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _exec_in_pod(*command: str, pod_name: str | None = None) -> subprocess.CompletedProcess[str]: + name = pod_name or _first_gpu_pod_name() + return run_kubectl("exec", "-n", NAMESPACE, name, "--", *command) + + +def _first_gpu_pod_name() -> str: + result = run_kubectl( + "get", "pods", "-n", NAMESPACE, "-l", APP_LABEL, + "--field-selector=status.phase=Running", "-o", "jsonpath={.items[0].metadata.name}", + ) + assert result.returncode == 0 and result.stdout.strip(), ( + f"No Running GPU pod found: {result.stdout}{result.stderr}" + ) + return result.stdout.strip() + + +def _node_allocatable(resource_key: str) -> int: + """Sum of allocatable across all nodes.""" + result = run_kubectl("get", "nodes", "-o", "json") + if result.returncode != 0: + return 0 + nodes = json.loads(result.stdout) + total = 0 + for node in nodes.get("items", []): + val = node.get("status", {}).get("allocatable", {}).get(resource_key, "0") + try: + total += int(val) + except ValueError: + pass + return total + + +# ── Test class ──────────────────────────────────────────────────────────────── + + +class TestMediaGpuTranscodingLane: + """ + GPU transcoding and hardware-passthrough lane. + + All tests are skipped at module import time when no GPU resource is + allocatable on any cluster node. See the module docstring for the full + substrate dependency chain (epic #54). + """ + + # ── 1. Substrate prerequisites ──────────────────────────────────────────── + + def test_gpu_node_has_allocatable_capacity(self): + """ + At least one node reports allocatable GPU resource (e.g. nvidia.com/gpu). + + This is the primary substrate gate: if this test fails, the device + plugin is either not installed or not reporting capacity correctly. + """ + capacity = _node_allocatable(GPU_RESOURCE_KEY) + assert capacity >= 1, ( + f"Expected at least 1 allocatable {GPU_RESOURCE_KEY} but got {capacity}. " + "Install the nvidia-device-plugin DaemonSet (see #54)." + ) + write_artifact("gpu-node-allocatable.txt", str(capacity)) + + def test_device_plugin_daemonset_is_running(self): + """ + The GPU device-plugin DaemonSet has at least one Running pod on the + GPU node. Without the device plugin, resource limits on pods are + silently rejected and no GPU is actually allocated. + """ + result = run_kubectl( + "get", "pods", "--all-namespaces", + "-l", DEVICE_PLUGIN_LABEL, + "-o", "json", + ) + assert result.returncode == 0, result.stdout + result.stderr + write_artifact("gpu-device-plugin-pods.json", result.stdout) + pods = json.loads(result.stdout) + running = [ + p for p in pods.get("items", []) + if p.get("status", {}).get("phase") == "Running" + ] + assert running, ( + f"No Running pods found with label '{DEVICE_PLUGIN_LABEL}'. " + "Verify nvidia-device-plugin DaemonSet is installed and healthy (see #54)." + ) + + # ── 2. Deployment ───────────────────────────────────────────────────────── + + def test_gpu_deployment_becomes_ready(self): + """ + Workload requesting GPU resource reaches availableReplicas >= 1. + + A pod that specifies 'resources.limits: nvidia.com/gpu: 1' will stay + Pending indefinitely if the device plugin is absent or if no node has + allocatable GPU capacity. This test confirms Kubernetes has scheduled + and started the pod with the resource limit satisfied. + """ + result = run_kubectl( + "get", "deployment", DEPLOYMENT_NAME, "-n", NAMESPACE, "-o", "json" + ) + assert result.returncode == 0, result.stdout + result.stderr + write_artifact("gpu-deployment.json", result.stdout) + data = json.loads(result.stdout) + status = data.get("status", {}) + assert status.get("availableReplicas", 0) >= 1, json.dumps(status, indent=2) + assert status.get("readyReplicas", 0) >= 1, json.dumps(status, indent=2) + + def test_gpu_pod_reaches_running_state(self): + """Pod with GPU resource limit reaches Running state.""" + pods = get_pods_json() + write_artifact("gpu-pods.json", json.dumps(pods, indent=2)) + items = pods.get("items", []) + assert items, "No GPU media pod found" + pod = items[0] + phase = pod.get("status", {}).get("phase") + assert phase == "Running", json.dumps(pod.get("status", {}), indent=2) + container_statuses = pod.get("status", {}).get("containerStatuses", []) + assert container_statuses, "No containerStatuses found" + for cs in container_statuses: + assert cs.get("ready"), f"Container {cs.get('name')} not ready: {cs}" + + def test_gpu_resource_limit_present_in_pod_spec(self): + """ + Pod spec carries an explicit GPU resource limit so Kubernetes enforces + device scheduling rather than silently running without GPU access. + """ + pods = get_pods_json() + items = pods.get("items", []) + assert items, "No GPU media pod found" + pod = items[0] + containers = pod.get("spec", {}).get("containers", []) + assert containers, "No containers in GPU pod spec" + found_limit = False + for container in containers: + limits = container.get("resources", {}).get("limits", {}) + if GPU_RESOURCE_KEY in limits: + found_limit = True + break + assert found_limit, ( + f"GPU resource limit '{GPU_RESOURCE_KEY}' not found in any container spec.\n" + + json.dumps([c.get("resources") for c in containers], indent=2) + ) + + # ── 3. Device visibility ────────────────────────────────────────────────── + + def test_gpu_device_node_visible_in_container(self): + """ + At least one NVIDIA device node (/dev/nvidia*) is visible inside the + container. This proves nvidia-container-toolkit mounted the device + through rather than the pod receiving an empty device list. + """ + result = _exec_in_pod("sh", "-c", "ls /dev/nvidia* 2>/dev/null || echo MISSING") + assert result.returncode == 0, result.stdout + result.stderr + write_artifact("gpu-dev-nodes.txt", result.stdout) + assert "MISSING" not in result.stdout, ( + "No /dev/nvidia* device nodes found inside the container. " + "Verify nvidia-container-toolkit is installed and the container " + "runtime is configured for GPU passthrough (see #54)." + ) + + def test_nvidia_smi_reports_gpu_in_container(self): + """ + nvidia-smi runs successfully inside the container and reports the GPU. + + This confirms the driver userspace library path is correctly mapped + into the container, not just the device node. + """ + result = _exec_in_pod("nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader") + write_artifact("gpu-nvidia-smi.txt", result.stdout + result.stderr) + assert result.returncode == 0, ( + "nvidia-smi failed inside container. " + "Check that the NVIDIA driver is loaded on the host and " + "nvidia-container-toolkit is configured (see #54).\n" + + result.stderr + ) + assert result.stdout.strip(), "nvidia-smi returned empty GPU list" + + # ── 4. Transcoding capability ───────────────────────────────────────────── + + def test_ffmpeg_lists_nvenc_encoder(self): + """ + ffmpeg reports the nvenc hardware encoder as available. + + This is the pre-flight check before any actual transcoding: if nvenc + is absent, the CUDA encoder stack is not functional regardless of + device visibility. + """ + result = _exec_in_pod("sh", "-c", "ffmpeg -encoders 2>&1 | grep nvenc || echo NVENC_MISSING") + write_artifact("gpu-ffmpeg-encoders.txt", result.stdout + result.stderr) + assert "NVENC_MISSING" not in result.stdout, ( + "nvenc encoder not listed by ffmpeg. " + "Verify the container image includes ffmpeg with CUDA/NVENC support " + "and the NVIDIA driver version is compatible." + ) + + def test_ffmpeg_hardware_transcode_completes(self): + """ + ffmpeg performs a minimal hardware-accelerated transcode using NVENC. + + Generates a 1-second synthetic video stream in software and re-encodes + it using the NVENC hardware encoder. A successful exit code confirms + the full encode pipeline works end-to-end: device access, driver + compatibility, and encoder initialization. + + This is the primary transcoding contract test. + """ + result = _exec_in_pod( + "sh", "-c", + "ffmpeg -hide_banner -loglevel error" + " -f lavfi -i testsrc=duration=1:size=1280x720:rate=30" + " -c:v h264_nvenc -preset fast -f null - 2>&1", + ) + write_artifact("gpu-transcode-output.txt", result.stdout + result.stderr) + assert result.returncode == 0, ( + "Hardware transcode failed. " + "Check that nvidia-smi and nvenc are working inside the container.\n" + + result.stdout + result.stderr + ) + + # ── 5. Resource lifecycle ───────────────────────────────────────────────── + + def test_gpu_resource_is_released_after_pod_deletion(self): + """ + After the GPU pod is deleted, the GPU resource appears allocatable on + the node again within 60 s. + + This test validates that the device plugin correctly tracks resource + release, which is a prerequisite for the GPU lane to be idempotent + (re-runnable without stranding GPU capacity). + """ + before = _node_allocatable(GPU_RESOURCE_KEY) + write_artifact("gpu-allocatable-before-delete.txt", str(before)) + + pod_name = _first_gpu_pod_name() + delete = run_kubectl("delete", "pod", pod_name, "-n", NAMESPACE, "--wait=false") + assert delete.returncode == 0, delete.stdout + delete.stderr + write_artifact("gpu-pod-delete.txt", delete.stdout + delete.stderr) + + # Wait for the pod to disappear and GPU capacity to restore + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + after = _node_allocatable(GPU_RESOURCE_KEY) + if after >= before: + write_artifact("gpu-allocatable-after-delete.txt", str(after)) + return + time.sleep(5) + + after = _node_allocatable(GPU_RESOURCE_KEY) + write_artifact("gpu-allocatable-after-delete.txt", str(after)) + assert after >= before, ( + f"GPU allocatable capacity did not recover: was {before}, now {after} " + "after pod deletion. Check device-plugin health." + ) + + # ── 6. Explicitly deferred paths ───────────────────────────────────────── + + def test_kubevirt_vm_gpu_passthrough_is_out_of_scope_for_this_lane(self): + """ + KubeVirt VM-backed GPU passthrough (VFIO/IOMMU path) is out of scope + for the base GPU transcoding lane. That path requires additional + substrate work tracked in #54 (homelab substrate epic). This lane + validates k8s-native container GPU access only (device-plugin + + nvidia-container-toolkit path). + """ + pytest.skip( + "KubeVirt VM-backed GPU passthrough deferred to #54; " + "requires VFIO/IOMMU substrate configuration — " + "out of scope for container-native GPU lane" + ) + + def test_multi_gpu_and_mig_slicing_is_out_of_scope_for_this_lane(self): + """ + Multi-GPU scheduling and MIG (Multi-Instance GPU) slicing are out of + scope for this lane. The lab has at most one GPU; MIG requires Ampere + or later. Tracked as a separate follow-up if multi-GPU hardware + becomes available. + """ + pytest.skip( + "Multi-GPU scheduling and MIG slicing deferred; " + "no multi-GPU hardware in current lab — out of scope for base GPU lane" + ) + + def test_amd_and_intel_gpu_paths_are_out_of_scope_for_this_lane(self): + """ + AMD ROCm and Intel QSV/VAAPI transcoding paths are out of scope for + this lane. Primary target is NVIDIA CUDA/NVENC. AMD and Intel iGPU + paths are deferred until the NVIDIA path is validated and hardware is + available. + """ + pytest.skip( + "AMD ROCm and Intel QSV/VAAPI transcoding deferred; " + "primary target is NVIDIA CUDA/NVENC — deferred until NVIDIA path is validated" + )