diff --git a/.github/actions/compute-version/action.yml b/.github/actions/compute-version/action.yml index 7db187ee..98b7a521 100644 --- a/.github/actions/compute-version/action.yml +++ b/.github/actions/compute-version/action.yml @@ -65,9 +65,9 @@ runs: HEAD_SHA="$(git rev-parse HEAD)" # Highest release tag that is a proper ancestor of HEAD. Two filters: - # * skip tags ON HEAD — keeps the computation stable even if - # release.yml already tagged this same push (avoids an empty commit - # range -> a false "none" bump). + # * skip tags ON HEAD — if release.yml already tagged this same push, + # we must still base off the PREVIOUS release, not HEAD's own tag, + # so re-running recomputes the same version instead of bumping again. # * skip tags NOT reachable from HEAD — so re-running an OLDER commit # after a newer release exists doesn't pick that newer, unrelated # tag as its base (it would compute a wrong/backwards version). @@ -84,10 +84,8 @@ runs: if [ -n "$LATEST" ]; then BASE="${LATEST#v}" - RANGE="${LATEST}..HEAD" else BASE="$FALLBACK" - RANGE="" fi # Detect the Conventional Commits bump level for a block of text. @@ -109,13 +107,11 @@ runs: # Squash merge: the PR title is the future main commit subject. BUMP="$(detect_bump "$PR_TITLE")" else - # Push/release: read the actual commits landed since the base. - if [ -n "$RANGE" ]; then - LOG="$(git log "$RANGE" --pretty=format:'%s%n%b%n--END--')" - else - LOG="$(git log --pretty=format:'%s%n%b%n--END--')" - fi - BUMP="$(detect_bump "$LOG")" + # Push/release: we squash-merge, so HEAD is the single commit that + # just landed on main and its SUBJECT is the PR title. Detect the + # bump from that subject ONLY. + SUBJECT="$(git log -1 --pretty=format:'%s')" + BUMP="$(detect_bump "$SUBJECT")" fi IFS='.' read -r MAJOR MINOR PATCH <<<"$BASE" diff --git a/.github/actions/docker-setup/action.yml b/.github/actions/docker-setup/action.yml index e036de15..5a307a7b 100644 --- a/.github/actions/docker-setup/action.yml +++ b/.github/actions/docker-setup/action.yml @@ -11,6 +11,9 @@ inputs: description: 'Maximum parallelism for Docker Build' required: false default: 8 + cache-key: + description: 'Sticky-disk cache lineage key. Use a distinct value per image family (e.g. base, pytorch, rocm) so families do not thrash each other''s BuildKit cache. Required by setup-docker-builder v2.' + required: true runs: using: 'composite' @@ -63,9 +66,10 @@ runs: password: ${{ inputs.dockerhub-token }} - name: Set up Docker Build - uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba + uses: useblacksmith/setup-docker-builder@1e75355f81cbb3f93cba1eb5dd5e5b05110ca8f9 # v2.0.1 with: max-parallelism: ${{ inputs.max-parallelism }} + cache-key: ${{ inputs.cache-key }} - name: Inspect buildx builder shell: bash diff --git a/.github/actions/grype/action.yml b/.github/actions/grype/action.yml index c7020a3a..42f915b5 100644 --- a/.github/actions/grype/action.yml +++ b/.github/actions/grype/action.yml @@ -1,5 +1,5 @@ name: Grype -description: "Scan Docker images with Grype. For now it's report-only and doesn't fail the build, until we fix the vulnerabilities." +description: "Scan Docker images with Grype. Report-only by default; set fail-on-findings to fail the build on CRITICAL/HIGH (fixed) vulnerabilities." inputs: image-refs: @@ -9,6 +9,10 @@ inputs: description: "Maximum number of Grype scans to run concurrently" required: false default: "2" + fail-on-findings: + description: "When 'true', fail the build if any CRITICAL/HIGH (fixed) vulnerabilities are found. When 'false' (default), only report them." + required: false + default: "false" runs: using: composite @@ -42,6 +46,7 @@ runs: env: IMAGE_REFS: ${{ inputs.image-refs }} MAX_PARALLEL_SCANS: ${{ inputs.max-parallel-scans }} + FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} run: | set -uo pipefail mapfile -t refs < <(echo "${IMAGE_REFS}" | jq -r '.[]') @@ -102,7 +107,10 @@ runs: if [ ${#failed[@]} -gt 0 ]; then echo "::error::Grype found CRITICAL/HIGH vulnerabilities in:" printf ' - %s\n' "${failed[@]}" - # exit 1 # TODO: Uncomment this when we fix the vulnerabilities + if [ "${FAIL_ON_FINDINGS}" = "true" ]; then + exit 1 + fi + echo "fail-on-findings is not enabled; reporting only." else echo "All images clean of CRITICAL/HIGH (fixed) vulnerabilities." fi diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index f62c4684..3119a2e1 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -45,6 +45,7 @@ jobs: with: dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + cache-key: ${{ github.repository }}/base - name: Build base images id: build @@ -71,6 +72,7 @@ jobs: uses: ./.github/actions/grype with: image-refs: ${{ steps.refs.outputs.refs }} + fail-on-findings: "true" - name: Push images uses: ./.github/actions/docker-push @@ -176,6 +178,7 @@ jobs: with: dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + cache-key: ${{ github.repository }}/autoresearch - name: Build autoresearch images id: build @@ -334,6 +337,7 @@ jobs: dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} max-parallelism: 6 + cache-key: ${{ github.repository }}/pytorch-${{ matrix.cuda }} - name: Build pytorch images (${{ matrix.cuda }}) id: build @@ -364,6 +368,7 @@ jobs: uses: ./.github/actions/grype with: image-refs: ${{ steps.refs.outputs.refs }} + fail-on-findings: "true" - name: Push images if: github.event_name != 'pull_request' || steps.changes.outputs.pytorch_any_changed == 'true' @@ -497,6 +502,7 @@ jobs: with: dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + cache-key: ${{ github.repository }}/cluster-${{ matrix.cuda }} - name: Build cluster images (${{ matrix.cuda }}) id: build diff --git a/.github/workflows/manual-release.yml b/.github/workflows/manual-release.yml new file mode 100644 index 00000000..2641cfba --- /dev/null +++ b/.github/workflows/manual-release.yml @@ -0,0 +1,78 @@ +name: Manual Release (break-glass) + +# BREAK-GLASS ONLY. Use this when a version's images were ALREADY built and +# pushed by a prior "Build and Release" run, but that run's `release` job got +# stuck in `skipped` — e.g. a flaky build failed and turned the strict +# release gate false, and GitHub's "Re-run failed jobs" never resurrects a +# job that ended up `skipped` (only `failure`/`cancelled` ones). See +# release.yml. This workflow does NOT build, scan, sign or push anything; it +# only creates the git tag + GitHub Release the stuck run failed to publish. +on: + workflow_dispatch: + inputs: + tag: + description: "Exact tag to publish, e.g. v1.1.0. Images for this version MUST already be built and pushed." + required: true + type: string + +permissions: + contents: read + +# Share the push-release lane from release.yml (group is a cross-workflow +# string) so a manual release can't run while a real push release is in +# flight for main. +concurrency: + group: "Build and Release-main-release" + cancel-in-progress: false + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Guard — main only + if: github.ref != 'refs/heads/main' + run: | + echo "::error::Manual release must be run from main (got ${{ github.ref }})" + exit 1 + + - name: Validate tag input + env: + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + if ! [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::tag must look like vMAJOR.MINOR.PATCH (got '$TAG')" + exit 1 + fi + + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Ensure tag does not already exist + env: + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + git fetch --tags --force --quiet + if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + echo "::error::tag ${TAG} already exists — nothing to do (release already published?)" + exit 1 + fi + + - name: Create tag and GitHub Release + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 + with: + tag_name: ${{ inputs.tag }} + name: ${{ inputs.tag }} + target_commitish: ${{ github.sha }} + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Summary + run: echo "Manually released ${{ inputs.tag }} at ${GITHUB_SHA} (images assumed already pushed)." diff --git a/.github/workflows/nvidia.yml b/.github/workflows/nvidia.yml index a154de71..8a5e8590 100644 --- a/.github/workflows/nvidia.yml +++ b/.github/workflows/nvidia.yml @@ -41,6 +41,7 @@ jobs: with: dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + cache-key: ${{ github.repository }}/nvidia - name: Build nvidia images (${{ matrix.nvidia }}) id: build @@ -69,6 +70,7 @@ jobs: uses: ./.github/actions/grype with: image-refs: ${{ steps.refs.outputs.refs }} + fail-on-findings: "true" - name: Push images uses: ./.github/actions/docker-push diff --git a/.github/workflows/rocm.yml b/.github/workflows/rocm.yml index 5d5f0209..9d84caed 100644 --- a/.github/workflows/rocm.yml +++ b/.github/workflows/rocm.yml @@ -41,6 +41,7 @@ jobs: with: dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + cache-key: ${{ github.repository }}/rocm - name: Build rocm images (${{ matrix.rocm }}) id: build diff --git a/official-templates/autoresearch/Dockerfile b/official-templates/autoresearch/Dockerfile index 9d32277b..292a0d87 100644 --- a/official-templates/autoresearch/Dockerfile +++ b/official-templates/autoresearch/Dockerfile @@ -4,8 +4,8 @@ FROM ${BASE_IMAGE} SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install runpodctl for pod management (scaling up GPUs) -ARG RUNPODCTL_VERSION=v2.3.0 -ARG RUNPODCTL_SHA256=908f2210571e8a26a1cba6fb45f09556b34dcad3e1b20dd502df2adf7a57c169 +ARG RUNPODCTL_VERSION=v2.7.2 +ARG RUNPODCTL_SHA256=8216810f9210c3e18c648d828f4d378cdaac0930389b460b1574ff2a3dfd1624 RUN wget -q --https-only --secure-protocol=TLSv1_2 --max-redirect=3 \ -O /tmp/runpodctl.tar.gz \ "https://github.com/runpod/runpodctl/releases/download/${RUNPODCTL_VERSION}/runpodctl-linux-amd64.tar.gz" && \ diff --git a/official-templates/base/Dockerfile b/official-templates/base/Dockerfile index 2905c6cc..3dd2bf31 100644 --- a/official-templates/base/Dockerfile +++ b/official-templates/base/Dockerfile @@ -109,18 +109,6 @@ RUN python -m pip install --upgrade --no-cache-dir -r requirements.txt && \ python /tmp/scrub-stale-metadata.py /requirements.txt && \ rm /tmp/scrub-stale-metadata.py -# Install filebrowser (pinned + SHA-256 verified, avoids `curl | bash`). -# Bump FILEBROWSER_VERSION + FILEBROWSER_SHA256 together when updating. -# Checksums: https://github.com/filebrowser/filebrowser/releases -ARG FILEBROWSER_VERSION=2.63.5 -ARG FILEBROWSER_SHA256=b36ad6296db0a749a5adbc792ab5321d11b307106123d44e171b7c158fcca2d9 -RUN curl -fsSL --proto '=https' --tlsv1.2 -o /tmp/filebrowser.tar.gz \ - "https://github.com/filebrowser/filebrowser/releases/download/v${FILEBROWSER_VERSION}/linux-amd64-filebrowser.tar.gz" && \ - echo "${FILEBROWSER_SHA256} /tmp/filebrowser.tar.gz" | sha256sum -c - && \ - tar -xzf /tmp/filebrowser.tar.gz -C /usr/local/bin filebrowser && \ - chmod +x /usr/local/bin/filebrowser && \ - rm /tmp/filebrowser.tar.gz - # NGINX Proxy COPY --from=proxy nginx.conf /etc/nginx/nginx.conf COPY --from=proxy snippets /etc/nginx/snippets diff --git a/official-templates/base/requirements.txt b/official-templates/base/requirements.txt index 571b00f1..32e0f60b 100644 --- a/official-templates/base/requirements.txt +++ b/official-templates/base/requirements.txt @@ -1,5 +1,5 @@ hf_transfer==0.1.9 -jupyterlab==4.5.9 +jupyterlab==4.5.10 ipywidgets==8.1.8 jupyter-archive==3.4.0 notebook==7.5.6 diff --git a/official-templates/nvidia-pytorch/requirements.txt b/official-templates/nvidia-pytorch/requirements.txt index 85f9de21..dc24a71d 100644 --- a/official-templates/nvidia-pytorch/requirements.txt +++ b/official-templates/nvidia-pytorch/requirements.txt @@ -2,13 +2,15 @@ aiohttp==3.14.1 black==26.3.1 jaraco.context==6.1.0 jupyter_server==2.20.0 -jupyterlab==4.5.9 -mistune==3.2.1 +jupyterlab==4.5.10 +mistune==3.3.0 +msgpack==1.2.1 nbconvert==7.17.1 notebook==7.5.6 onnx==1.21.0 -pillow==12.2.0 +pillow==12.3.0 protobuf==6.33.5 +soupsieve==2.8.4 tornado==6.5.7 urllib3==2.7.0 wheel==0.46.2 diff --git a/official-templates/pytorch-cluster/Dockerfile b/official-templates/pytorch-cluster/Dockerfile index 3f723334..8f92e888 100644 --- a/official-templates/pytorch-cluster/Dockerfile +++ b/official-templates/pytorch-cluster/Dockerfile @@ -39,8 +39,8 @@ RUN apt-get update --yes && \ rm -rf /var/lib/apt/lists/* # node_exporter — system metrics (pinned + SHA-256 verified). -ARG NODE_EXPORTER_VERSION=1.11.1 -ARG NODE_EXPORTER_SHA256=9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810 +ARG NODE_EXPORTER_VERSION=1.12.1 +ARG NODE_EXPORTER_SHA256=b51d8a76aa2a9156a55d501aca6276fae09e262259a5e4e831d2c2222f084e63 RUN curl -fsSL --proto '=https' --tlsv1.2 -o /tmp/node_exporter.tar.gz \ "https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz" && \ echo "${NODE_EXPORTER_SHA256} /tmp/node_exporter.tar.gz" | sha256sum -c - && \ @@ -62,9 +62,9 @@ RUN curl -fsSL --proto '=https' --tlsv1.2 -o /tmp/prometheus.tar.gz \ rm -rf /tmp/prometheus* # Grafana OSS — dashboards (pinned + SHA-256 verified). -ARG GRAFANA_VERSION=13.1.0 -ARG GRAFANA_BUILD=28013217238 -ARG GRAFANA_SHA256=4f562bb224b8bb758b47789381babb284cb41687da8d714f2ff0e118e945e775 +ARG GRAFANA_VERSION=13.1.1 +ARG GRAFANA_BUILD=29761037902 +ARG GRAFANA_SHA256=e47443214da0de041ffb29633d0977ce31ba7c8c569f09974ef5294a8ce32f08 RUN curl -fsSL --proto '=https' --tlsv1.2 -o /tmp/grafana.tar.gz \ "https://dl.grafana.com/grafana/release/${GRAFANA_VERSION}/grafana_${GRAFANA_VERSION}_${GRAFANA_BUILD}_linux_amd64.tar.gz" && \ echo "${GRAFANA_SHA256} /tmp/grafana.tar.gz" | sha256sum -c - && \ diff --git a/official-templates/pytorch/Dockerfile b/official-templates/pytorch/Dockerfile index f0c725b6..f753fe18 100644 --- a/official-templates/pytorch/Dockerfile +++ b/official-templates/pytorch/Dockerfile @@ -5,6 +5,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG WHEEL_SRC ARG TORCH +ARG PILLOW_VERSION # TORCH is a space-separated list of pip specs (e.g. "torch==2.8.0 torchvision==0.23.0 # torchaudio==2.8.0"). `read -ra` splits it into a real bash array so pip receives one @@ -18,3 +19,11 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ --upgrade \ "${TORCH_PKGS[@]}" \ --index-url "https://download.pytorch.org/whl/cu${WHEEL_SRC}" + +# torchvision pulls Pillow from the PyTorch wheel index, which lags PyPI and +# ships releases with known HIGH CVEs. Re-install a patched Pillow from PyPI, +# but only when it's already present (torch builds without torchvision have none). +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + if python -m pip show pillow > /dev/null 2>&1; then \ + python -m pip install --upgrade "pillow==${PILLOW_VERSION}"; \ + fi diff --git a/official-templates/pytorch/docker-bake.hcl b/official-templates/pytorch/docker-bake.hcl index f52be3cf..5284f0fc 100644 --- a/official-templates/pytorch/docker-bake.hcl +++ b/official-templates/pytorch/docker-bake.hcl @@ -1,5 +1,12 @@ # https://pytorch.org/get-started/locally/ +# Patched Pillow pulled from PyPI to override the outdated (CVE-affected) copy +# that torchvision drags in from the PyTorch wheel index. Keep in sync with the +# pin used by the nvidia-pytorch / rocm requirements.txt files. +variable "PILLOW_VERSION" { + default = "12.3.0" +} + variable "TORCH_META" { default = { "2.9.1" = {} @@ -118,6 +125,7 @@ target "pytorch-matrix" { BASE_IMAGE = "runpod/base:${RELEASE_VERSION}${RELEASE_SUFFIX}-cuda${build.cuda_code}-${build.ubuntu_name}" WHEEL_SRC = build.wheel_src TORCH = "torch==${build.torch}${build.torch_vision != "" ? " torchvision==${build.torch_vision}" : ""} torchaudio==${build.torch}" + PILLOW_VERSION = PILLOW_VERSION } tags = [ diff --git a/official-templates/rocm/requirements.txt b/official-templates/rocm/requirements.txt index 9223c639..9dedc340 100644 --- a/official-templates/rocm/requirements.txt +++ b/official-templates/rocm/requirements.txt @@ -3,11 +3,12 @@ PyJWT==2.13.0 aiohttp==3.14.1 cryptography==48.0.1 jaraco.context==6.1.0 -jupyterlab==4.5.9 +jupyterlab==4.5.10 lxml==6.1.0 +msgpack==1.2.1 notebook==7.5.6 onnx==1.21.0 -pillow==12.2.0 +pillow==12.3.0 protobuf==6.33.5 tornado==6.5.7 urllib3==2.7.0