diff --git a/.github/workflows/build-tromso-multirunner.yml b/.github/workflows/build-tromso-multirunner.yml index 917954e2..3bf686a2 100644 --- a/.github/workflows/build-tromso-multirunner.yml +++ b/.github/workflows/build-tromso-multirunner.yml @@ -173,39 +173,26 @@ jobs: mkdir -p ~/.cache/buildstream REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}-core:latest" -o .; then - if [ -f cas-core.tar.zst ]; then - # --exclude=sources: each pushed tarball is a full - # ~/.cache/buildstream snapshot (see the archive step in - # tuna-os/bst-ci's multirunner-build.yml), source downloads - # included. build_final only needs the built artifacts (cas/) - # to assemble final_target — every element it references is - # already cached, so it never re-stages sources for them. - # Merging 11 full per-chunk sources/ trees onto one small - # runner disk is what exhausted it in run 33295159787 - # ("No space left on device" mid-extract); skipping sources/ - # here cuts what actually hits disk to the artifacts we use. - zstd -d -c -T0 cas-core.tar.zst | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream" - rm -f cas-core.tar.zst - fi - fi + # Streamed, never staged: `oras pull -o .` wrote each tarball to the + # runner disk next to the cache it was unpacking into, and build_final + # in run 36971374464 died in this step with "No space left on device". + # scripts/cas-restore-stream.sh pipes the blob through zstd and tar + # (sources/ excluded) and resumes GHCR's mid-stream PROTOCOL_ERROR + # cuts. A missing chunk costs rebuild time here, never correctness. + BASE="ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}" + scripts/cas-restore-stream.sh "${BASE}-core:latest" \ + || echo "::warning::core CAS unavailable — build_final rebuilds it" + df -h / echo "$MATRIX_KEYS" | jq -r '.[]' | while read -r chunk; do CACHE_KEY=$(echo "$CACHE_KEYS" | jq -r --arg c "$chunk" '.[$c] // empty') - PULLED=false - if [ -n "$CACHE_KEY" ]; then - if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}-${chunk}:${CACHE_KEY}" -o .; then - PULLED=true - fi - fi - if [ "$PULLED" = "false" ]; then - oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}-${chunk}:latest" -o . || continue - fi - TARBALL="cas-${chunk}.tar.zst" - if [ -f "$TARBALL" ]; then - zstd -d -c -T0 "$TARBALL" | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream" - rm -f "$TARBALL" + if [ -n "$CACHE_KEY" ] \ + && scripts/cas-restore-stream.sh "${BASE}-${chunk}:${CACHE_KEY}"; then + : + elif ! scripts/cas-restore-stream.sh "${BASE}-${chunk}:latest"; then + echo "::warning::no CAS for ${chunk} — build_final rebuilds it" fi + df -h / done du -sh ~/.cache/buildstream @@ -248,6 +235,23 @@ jobs: if: always() run: rm -f client.key client.crt + - name: Check out the artifact and free the CAS + env: + BST_FLAGS: --no-interactive --config /src/buildstream-ci.conf + run: | + # The merged CAS and the squashed image do not fit on one runner + # together (xfce-linux#159 hit exactly this). Take the artifact out + # of the cache, then drop the cache; export works from .build-out. + df -h / + rm -rf .build-out + just bst artifact checkout \ + "${{ needs.multirunner.outputs.final_target }}" --directory /src/.build-out + du -sh .build-out + # Removed inside the bst container: its root owns the cache tree. + podman run --rm -v "${HOME}/.cache/buildstream:/cache:rw" "$BST2_IMAGE" \ + sh -c 'rm -rf /cache/cas /cache/artifacts /cache/sources' + df -h / + - name: Capture build timestamp id: timestamp run: echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" @@ -260,6 +264,8 @@ jobs: OCI_IMAGE_REVISION: ${{ github.sha }} OCI_IMAGE_VERSION: latest OCI_IMAGE_CREATED: ${{ steps.timestamp.outputs.created }} + # The CAS is gone by now (previous step); .build-out is the artifact. + EXPORT_REUSE_CHECKOUT: '1' run: | just export echo "image_ref=${{ env.IMAGE_NAME }}:latest" >> "$GITHUB_OUTPUT" @@ -445,33 +451,26 @@ jobs: mkdir -p ~/.cache/buildstream REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}-core:latest" -o .; then - if [ -f cas-core.tar.zst ]; then - # --exclude=sources: see the x86_64 build_final job above — - # build_final only needs cas/ to assemble final_target, and - # merging every chunk's full sources/ tree onto one runner - # disk is what exhausted it in run 33295159787. - zstd -d -c -T0 cas-core.tar.zst | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream" - rm -f cas-core.tar.zst - fi - fi + # Streamed, never staged: `oras pull -o .` wrote each tarball to the + # runner disk next to the cache it was unpacking into, and build_final + # in run 36971374464 died in this step with "No space left on device". + # scripts/cas-restore-stream.sh pipes the blob through zstd and tar + # (sources/ excluded) and resumes GHCR's mid-stream PROTOCOL_ERROR + # cuts. A missing chunk costs rebuild time here, never correctness. + BASE="ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}" + scripts/cas-restore-stream.sh "${BASE}-core:latest" \ + || echo "::warning::core CAS unavailable — build_final rebuilds it" + df -h / echo "$MATRIX_KEYS" | jq -r '.[]' | while read -r chunk; do CACHE_KEY=$(echo "$CACHE_KEYS" | jq -r --arg c "$chunk" '.[$c] // empty') - PULLED=false - if [ -n "$CACHE_KEY" ]; then - if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}-${chunk}:${CACHE_KEY}" -o .; then - PULLED=true - fi - fi - if [ "$PULLED" = "false" ]; then - oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}-${chunk}:latest" -o . || continue - fi - TARBALL="cas-${chunk}.tar.zst" - if [ -f "$TARBALL" ]; then - zstd -d -c -T0 "$TARBALL" | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream" - rm -f "$TARBALL" + if [ -n "$CACHE_KEY" ] \ + && scripts/cas-restore-stream.sh "${BASE}-${chunk}:${CACHE_KEY}"; then + : + elif ! scripts/cas-restore-stream.sh "${BASE}-${chunk}:latest"; then + echo "::warning::no CAS for ${chunk} — build_final rebuilds it" fi + df -h / done du -sh ~/.cache/buildstream diff --git a/Justfile b/Justfile index b6055b7a..aea110bd 100644 --- a/Justfile +++ b/Justfile @@ -45,9 +45,18 @@ export: if [ "$(id -u)" -ne 0 ]; then SUDO_CMD="sudo" fi - echo "==> Exporting Tromso OCI image..." - rm -rf .build-out - just bst artifact checkout oci/tromso.bst --directory /src/.build-out + # CI checks the artifact out in its own step and deletes the CAS before + # calling this recipe: the merged cache and the squashed image do not fit + # on a GitHub runner together. With EXPORT_REUSE_CHECKOUT=1, .build-out is + # already the artifact and must not be re-requested from BuildStream. + if [ "${EXPORT_REUSE_CHECKOUT:-0}" = "1" ]; then + [ -d .build-out ] || { echo "ERROR: EXPORT_REUSE_CHECKOUT=1 but .build-out is missing" >&2; exit 1; } + echo "==> Reusing the existing .build-out checkout..." + else + echo "==> Exporting Tromso OCI image..." + rm -rf .build-out + just bst artifact checkout oci/tromso.bst --directory /src/.build-out + fi echo "==> Loading and squashing OCI image..." IMAGE_ID=$($SUDO_CMD podman pull -q oci:.build-out) rm -rf .build-out diff --git a/scripts/cas-restore-stream.sh b/scripts/cas-restore-stream.sh new file mode 100755 index 00000000..de93e76d --- /dev/null +++ b/scripts/cas-restore-stream.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Stream a BuildStream CAS tarball from GHCR straight into the local cache. +# +# Usage: scripts/cas-restore-stream.sh [] +# e.g. ghcr.io/tuna-os/tromso/cache-tromso-core:latest +# defaults to ~/.cache/buildstream +# +# Why streaming: `oras pull ... -o .` first writes the whole tarball (up to +# ~28 GB compressed per chunk, 2026-09-10 GHCR sizes) to the runner disk and +# only then extracts it. On a ~100-110 GB GitHub runner that temp file is +# what pushed build_final over the edge in scheduled runs 34072176433, +# 34176106533 and 34298830711 ("No space left on device" from the runner +# itself). `oras blob fetch --output -` pipes the single tarball layer +# through zstd and tar instead, so the compressed bytes never touch disk. +# +# The CAS tarballs are single-layer artifacts (mediaType +# application/vnd.buildstream.cas.tar.zst, title cas-.tar.zst) pushed +# by tuna-os/bst-ci, so `.layers[0].digest` is the blob to fetch. +# +# --exclude=sources: each tarball is a full ~/.cache/buildstream snapshot, +# source downloads included. build_final only needs the built artifacts to +# assemble final_target, and merging every chunk's sources/ tree onto one +# runner disk is what exhausted it in run 33295159787. +# +# Exit status: 0 on success, non-zero if the manifest or blob could not be +# fetched or the extraction failed. Callers decide whether that is fatal. +set -euo pipefail + +REF="${1:?usage: $0 []}" +CACHE_DIR="${2:-${HOME}/.cache/buildstream}" + +mkdir -p "$CACHE_DIR" + +if ! MANIFEST=$(oras manifest fetch "$REF" 2>&1); then + echo "cas-restore-stream: no manifest for ${REF}: ${MANIFEST}" >&2 + exit 1 +fi + +DIGEST=$(printf '%s' "$MANIFEST" | jq -r '.layers[0].digest // empty') +SIZE=$(printf '%s' "$MANIFEST" | jq -r '.layers[0].size // 0') +if [ -z "$DIGEST" ]; then + echo "cas-restore-stream: ${REF} has no layers" >&2 + exit 1 +fi + +# The ref may already carry a tag; oras accepts :@. +echo "cas-restore-stream: streaming ${REF}@${DIGEST} ($((SIZE / 1024 / 1024)) MiB compressed) into ${CACHE_DIR}" +# ghcr-blob-stream.py rather than `oras blob fetch --output -`: GHCR cuts +# these multi-GB downloads with "PROTOCOL_ERROR" every few minutes and oras +# cannot resume, so a cut threw the whole restore away. The helper resumes +# with HTTP Range and verifies the digest. +python3 "$(dirname "$0")/ghcr-blob-stream.py" "$REF" "$DIGEST" "$SIZE" \ + | zstd -d -T0 \ + | tar --exclude=sources -xf - -C "$CACHE_DIR" diff --git a/scripts/ghcr-blob-stream.py b/scripts/ghcr-blob-stream.py new file mode 100755 index 00000000..af474bfc --- /dev/null +++ b/scripts/ghcr-blob-stream.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Stream one GHCR blob to stdout, resuming with HTTP Range after a cut. + +Usage: ghcr-blob-stream.py + e.g. ghcr.io/tuna-os/xfce-linux/cache-xfce-linux-core:latest + sha256:... of the blob (from the manifest's layer) + blob size in bytes (from the manifest's layer) + +Why: `oras blob fetch --output -` has no resume. GHCR's blob downloads of +the 10+ GB CAS tarballs get cut with "stream error: stream ID 1; +PROTOCOL_ERROR" a few minutes in (runs 36973935390 and 36805045437, always +on a 5-minute wall-clock boundary), and every cut threw the whole restore +away, so jobs rebuilt from scratch. This reconnects at the byte offset it +reached, and checks the sha256 of the full stream before exiting 0. + +Credentials come from the docker/oras auth file written by `oras login` +(~/.docker/config.json), falling back to an anonymous token. +""" + +import base64 +import hashlib +import http.client +import json +import os +import sys +import time +import urllib.error +import urllib.request + +CHUNK = 4 * 1024 * 1024 +MAX_ATTEMPTS = 40 + + +def log(msg: str) -> None: + print(f"ghcr-blob-stream: {msg}", file=sys.stderr, flush=True) + + +def basic_auth(registry: str) -> str | None: + for path in ( + os.environ.get("DOCKER_CONFIG", "") and os.path.join(os.environ["DOCKER_CONFIG"], "config.json"), + os.path.expanduser("~/.docker/config.json"), + os.path.join(os.environ.get("XDG_RUNTIME_DIR", "/nonexistent"), "containers/auth.json"), + ): + if not path or not os.path.isfile(path): + continue + try: + auths = json.load(open(path)).get("auths", {}) + except (OSError, ValueError): + continue + for key in (registry, f"https://{registry}"): + if auths.get(key, {}).get("auth"): + return auths[key]["auth"] + return None + + +def bearer(registry: str, repo: str) -> str: + req = urllib.request.Request(f"https://{registry}/token?service={registry}&scope=repository:{repo}:pull") + auth = basic_auth(registry) + if auth: + req.add_header("Authorization", f"Basic {auth}") + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp)["token"] + + +def main() -> int: + if len(sys.argv) != 4: + print(__doc__, file=sys.stderr) + return 2 + ref, digest, size = sys.argv[1], sys.argv[2], int(sys.argv[3]) + registry, _, rest = ref.partition("/") + repo = rest.rsplit("@", 1)[0] + if ":" in repo.rsplit("/", 1)[-1]: + repo = repo.rsplit(":", 1)[0] + url = f"https://{registry}/v2/{repo}/blobs/{digest}" + + out = sys.stdout.buffer + sha = hashlib.sha256() + offset = 0 + attempt = 0 + token = bearer(registry, repo) + while offset < size: + attempt += 1 + if attempt > MAX_ATTEMPTS: + log(f"giving up after {MAX_ATTEMPTS} attempts at byte {offset}/{size}") + return 1 + req = urllib.request.Request(url) + # Not forwarded on the redirect to the signed storage URL. + req.add_unredirected_header("Authorization", f"Bearer {token}") + if offset: + req.add_header("Range", f"bytes={offset}-") + try: + with urllib.request.urlopen(req, timeout=120) as resp: + if offset and resp.status != 206: + log(f"server ignored Range (HTTP {resp.status}); cannot resume") + return 1 + while offset < size: + buf = resp.read(min(CHUNK, size - offset)) + if not buf: + break + out.write(buf) + sha.update(buf) + offset += len(buf) + except urllib.error.HTTPError as exc: + if exc.code == 401: + token = bearer(registry, repo) + log(f"HTTP {exc.code} at byte {offset}/{size} (attempt {attempt}); retrying") + time.sleep(min(30, 2 * attempt)) + continue + except (OSError, ValueError, http.client.HTTPException) as exc: + log(f"connection cut at byte {offset}/{size} (attempt {attempt}): {exc}; resuming") + time.sleep(min(30, 2 * attempt)) + continue + if offset < size: + log(f"short read at byte {offset}/{size} (attempt {attempt}); resuming") + out.flush() + if offset != size: + log(f"read {offset} bytes, expected {size}") + return 1 + got = "sha256:" + sha.hexdigest() + if got != digest: + log(f"digest mismatch: got {got}, expected {digest}") + return 1 + log(f"done: {size} bytes in {attempt} attempt(s), digest verified") + return 0 + + +if __name__ == "__main__": + sys.exit(main())