Repository navigation
Expand file tree
/
Copy pathDockerfile.v100
More file actions
74 lines (67 loc) · 3.97 KB
/
Copy pathDockerfile.v100
File metadata and controls
74 lines (67 loc) · 3.97 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# V100-specific variant of the main Dockerfile (see Dockerfile itself for the
# shared rationale/usage). ONLY difference: forces a torch build against CUDA
# 12.6 instead of whatever the main resolve pulls (currently torch==2.13.0+cu130).
#
# Why this exists (2026-09-18): confirmed on a real V100 pod that
# torch==2.13.0+cu130 (pulled in transitively, no hard pin of our own) does
# NOT include compiled kernels for compute capability 7.0 (Volta/V100) at
# all -- torch.cuda.get_arch_list() only covers sm_75 and up, and every real
# task attempted on a V100 failed with
# `torch.AcceleratorError: no kernel image is available for execution on the
# device`. PyTorch's own warning at runtime names the fix: a build against an
# older CUDA toolkit (12.6) still includes sm_70. User's call: maintain this
# as a separate image rather than abandon V100 (~7 GPUs of otherwise-idle
# capacity).
#
# Build and push:
# docker build --platform linux/amd64 -f Dockerfile.v100 -t <registry>/<you>/ramanbench:v1-v100 .
# docker push <registry>/<you>/ramanbench:v1-v100
# Then point a V100-only k8s profile's `image:` at that tag (node_selector
# gpu: v100) -- NOT the main `:v1` tag, which stays on the default (newer,
# faster) cu130 build for A100/H100/H200/B200.
FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
# build-essential: same reason as the main Dockerfile -- torch>=2.13 (still
# true here, just a different CUDA build of the same 2.13.0 release) pulls in
# a triton version whose kernels are JIT-compiled at call time via
# `triton.runtime.build`, which shells out to `cc`. Missing here until a real
# smoke test caught it (a single-target ILTM job failed before even reaching
# that path, on the missing tabarena import below -- but this would have been
# the next failure once that was fixed).
RUN apt-get update && apt-get install -y --no-install-recommends \
git \
build-essential \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Same two-step install as the main Dockerfile (see its own comment for the
# full rationale) -- missing here until a real smoke test caught it: every
# task on this image failed with "ModuleNotFoundError: No module named
# 'tabarena'" (confirmed live via a real single-target ILTM job on an actual
# V100 pod), since this file never had the requirements-tabarena-git.txt /
# requirements-models-git.txt install steps the main Dockerfile has.
COPY pyproject.toml README.md requirements-tabarena-git.txt requirements-models-git.txt ./
COPY src ./src
RUN pip install --no-cache-dir --pre -e ".[models,benchmark]" \
&& pip install --no-cache-dir --pre -r requirements-tabarena-git.txt \
&& pip install --no-cache-dir --pre -r requirements-models-git.txt \
&& pip uninstall -y torchaudio
# The only intentional difference from the main Dockerfile: force torch onto a
# cu126 build (V100/Volta-compatible) after the main resolve already picked
# whatever CUDA variant it wanted. Same torch version (2.13.0) as the main
# image -- only the bundled CUDA kernels differ -- so this doesn't change
# which API surface autogluon/tabarena/etc. see, only which GPU archs the
# actual kernels support. Placed AFTER the tabarena/models-git installs above
# (not before) so neither of those can pull torch back onto its own default
# cu130 resolve as a transitive dependency.
#
# --force-reinstall is required, not optional: without it, pip sees
# "torch==2.13.0" as already satisfied by the installed "2.13.0+cu130" (pip
# matches on the release segment, ignoring the local version/build tag) and
# silently skips the reinstall entirely -- confirmed real: the first build of
# this image without --force-reinstall still had cu130 torch, verified via a
# real V100 pod (torch.cuda.get_arch_list() still missing sm_70/CC 7.0).
RUN pip install --no-cache-dir --force-reinstall --index-url https://download.pytorch.org/whl/cu126 torch==2.13.0
COPY scripts ./scripts
COPY cluster ./cluster
COPY configs ./configs
RUN chmod +x cluster/k8s_entrypoint.sh
ENTRYPOINT ["/bin/bash", "cluster/k8s_entrypoint.sh"]