hanthor · Oct 2, 2026 · Oct 2, 2026
@@ -173,39 +173,26 @@ jobs:
           mkdir -p ~/.cache/buildstream
           REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
 
-          if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}-core:latest" -o .; then
-            if [ -f cas-core.tar.zst ]; then
-              # --exclude=sources: each pushed tarball is a full
-              # ~/.cache/buildstream snapshot (see the archive step in
-              # tuna-os/bst-ci's multirunner-build.yml), source downloads
-              # included. build_final only needs the built artifacts (cas/)
-              # to assemble final_target — every element it references is
-              # already cached, so it never re-stages sources for them.
-              # Merging 11 full per-chunk sources/ trees onto one small
-              # runner disk is what exhausted it in run 33295159787
-              # ("No space left on device" mid-extract); skipping sources/
-              # here cuts what actually hits disk to the artifacts we use.
-              zstd -d -c -T0 cas-core.tar.zst | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream"
-              rm -f cas-core.tar.zst
-            fi
-          fi
+          # Streamed, never staged: `oras pull -o .` wrote each tarball to the
+          # runner disk next to the cache it was unpacking into, and build_final
+          # in run 36971374464 died in this step with "No space left on device".
+          # scripts/cas-restore-stream.sh pipes the blob through zstd and tar
+          # (sources/ excluded) and resumes GHCR's mid-stream PROTOCOL_ERROR
+          # cuts. A missing chunk costs rebuild time here, never correctness.
+          BASE="ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}"
+          scripts/cas-restore-stream.sh "${BASE}-core:latest" \
+            || echo "::warning::core CAS unavailable — build_final rebuilds it"
+          df -h /
 
           echo "$MATRIX_KEYS" | jq -r '.[]' | while read -r chunk; do
             CACHE_KEY=$(echo "$CACHE_KEYS" | jq -r --arg c "$chunk" '.[$c] // empty')
-            PULLED=false
-            if [ -n "$CACHE_KEY" ]; then
-              if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}-${chunk}:${CACHE_KEY}" -o .; then
-                PULLED=true
-              fi
-            fi
-            if [ "$PULLED" = "false" ]; then
-              oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}-${chunk}:latest" -o . || continue
-            fi
-            TARBALL="cas-${chunk}.tar.zst"
-            if [ -f "$TARBALL" ]; then
-              zstd -d -c -T0 "$TARBALL" | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream"
-              rm -f "$TARBALL"
+            if [ -n "$CACHE_KEY" ] \
+               && scripts/cas-restore-stream.sh "${BASE}-${chunk}:${CACHE_KEY}"; then
+              :
+            elif ! scripts/cas-restore-stream.sh "${BASE}-${chunk}:latest"; then
+              echo "::warning::no CAS for ${chunk} — build_final rebuilds it"
             fi
+            df -h /
           done
           du -sh ~/.cache/buildstream
 
@@ -248,6 +235,23 @@ jobs:
         if: always()
         run: rm -f client.key client.crt
 
+      - name: Check out the artifact and free the CAS
+        env:
+          BST_FLAGS: --no-interactive --config /src/buildstream-ci.conf
+        run: |
+          # The merged CAS and the squashed image do not fit on one runner
+          # together (xfce-linux#159 hit exactly this). Take the artifact out
+          # of the cache, then drop the cache; export works from .build-out.
+          df -h /
+          rm -rf .build-out
+          just bst artifact checkout \
+            "${{ needs.multirunner.outputs.final_target }}" --directory /src/.build-out
+          du -sh .build-out
+          # Removed inside the bst container: its root owns the cache tree.
+          podman run --rm -v "${HOME}/.cache/buildstream:/cache:rw" "$BST2_IMAGE" \
+            sh -c 'rm -rf /cache/cas /cache/artifacts /cache/sources'
+          df -h /
+
       - name: Capture build timestamp
         id: timestamp
         run: echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
@@ -260,6 +264,8 @@ jobs:
           OCI_IMAGE_REVISION: ${{ github.sha }}
           OCI_IMAGE_VERSION: latest
           OCI_IMAGE_CREATED: ${{ steps.timestamp.outputs.created }}
+          # The CAS is gone by now (previous step); .build-out is the artifact.
+          EXPORT_REUSE_CHECKOUT: '1'
         run: |
           just export
           echo "image_ref=${{ env.IMAGE_NAME }}:latest" >> "$GITHUB_OUTPUT"
@@ -445,33 +451,26 @@ jobs:
           mkdir -p ~/.cache/buildstream
           REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
 
-          if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}-core:latest" -o .; then
-            if [ -f cas-core.tar.zst ]; then
-              # --exclude=sources: see the x86_64 build_final job above —
-              # build_final only needs cas/ to assemble final_target, and
-              # merging every chunk's full sources/ tree onto one runner
-              # disk is what exhausted it in run 33295159787.
-              zstd -d -c -T0 cas-core.tar.zst | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream"
-              rm -f cas-core.tar.zst
-            fi
-          fi
+          # Streamed, never staged: `oras pull -o .` wrote each tarball to the
+          # runner disk next to the cache it was unpacking into, and build_final
+          # in run 36971374464 died in this step with "No space left on device".
+          # scripts/cas-restore-stream.sh pipes the blob through zstd and tar
+          # (sources/ excluded) and resumes GHCR's mid-stream PROTOCOL_ERROR
+          # cuts. A missing chunk costs rebuild time here, never correctness.
+          BASE="ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}"
+          scripts/cas-restore-stream.sh "${BASE}-core:latest" \
+            || echo "::warning::core CAS unavailable — build_final rebuilds it"
+          df -h /
 
           echo "$MATRIX_KEYS" | jq -r '.[]' | while read -r chunk; do
             CACHE_KEY=$(echo "$CACHE_KEYS" | jq -r --arg c "$chunk" '.[$c] // empty')
-            PULLED=false
-            if [ -n "$CACHE_KEY" ]; then
-              if oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}-${chunk}:${CACHE_KEY}" -o .; then
-                PULLED=true
-              fi
-            fi
-            if [ "$PULLED" = "false" ]; then
-              oras pull "ghcr.io/${REPO_LOWER}/cache-${IMAGE_NAME}${ARCH_SUFFIX}-${chunk}:latest" -o . || continue
-            fi
-            TARBALL="cas-${chunk}.tar.zst"
-            if [ -f "$TARBALL" ]; then
-              zstd -d -c -T0 "$TARBALL" | tar --exclude=sources -xf - -C "$HOME/.cache/buildstream"
-              rm -f "$TARBALL"
+            if [ -n "$CACHE_KEY" ] \
+               && scripts/cas-restore-stream.sh "${BASE}-${chunk}:${CACHE_KEY}"; then
+              :
+            elif ! scripts/cas-restore-stream.sh "${BASE}-${chunk}:latest"; then
+              echo "::warning::no CAS for ${chunk} — build_final rebuilds it"
             fi
+            df -h /
           done
           du -sh ~/.cache/buildstream
 

@@ -45,9 +45,18 @@ export:
     if [ "$(id -u)" -ne 0 ]; then
         SUDO_CMD="sudo"
     fi
-    echo "==> Exporting Tromso OCI image..."
-    rm -rf .build-out
-    just bst artifact checkout oci/tromso.bst --directory /src/.build-out
+    # CI checks the artifact out in its own step and deletes the CAS before
+    # calling this recipe: the merged cache and the squashed image do not fit
+    # on a GitHub runner together. With EXPORT_REUSE_CHECKOUT=1, .build-out is
+    # already the artifact and must not be re-requested from BuildStream.
+    if [ "${EXPORT_REUSE_CHECKOUT:-0}" = "1" ]; then
+        [ -d .build-out ] || { echo "ERROR: EXPORT_REUSE_CHECKOUT=1 but .build-out is missing" >&2; exit 1; }
+        echo "==> Reusing the existing .build-out checkout..."
+    else
+        echo "==> Exporting Tromso OCI image..."
+        rm -rf .build-out
+        just bst artifact checkout oci/tromso.bst --directory /src/.build-out
+    fi
     echo "==> Loading and squashing OCI image..."
     IMAGE_ID=$($SUDO_CMD podman pull -q oci:.build-out)
     rm -rf .build-out

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Stream a BuildStream CAS tarball from GHCR straight into the local cache.
+#
+# Usage: scripts/cas-restore-stream.sh <oras-ref> [<cache-dir>]
+#   <oras-ref>   e.g. ghcr.io/tuna-os/tromso/cache-tromso-core:latest
+#   <cache-dir>  defaults to ~/.cache/buildstream
+#
+# Why streaming: `oras pull ... -o .` first writes the whole tarball (up to
+# ~28 GB compressed per chunk, 2026-09-10 GHCR sizes) to the runner disk and
+# only then extracts it. On a ~100-110 GB GitHub runner that temp file is
+# what pushed build_final over the edge in scheduled runs 34072176433,
+# 34176106533 and 34298830711 ("No space left on device" from the runner
+# itself). `oras blob fetch --output -` pipes the single tarball layer
+# through zstd and tar instead, so the compressed bytes never touch disk.
+#
+# The CAS tarballs are single-layer artifacts (mediaType
+# application/vnd.buildstream.cas.tar.zst, title cas-<name>.tar.zst) pushed
+# by tuna-os/bst-ci, so `.layers[0].digest` is the blob to fetch.
+#
+# --exclude=sources: each tarball is a full ~/.cache/buildstream snapshot,
+# source downloads included. build_final only needs the built artifacts to
+# assemble final_target, and merging every chunk's sources/ tree onto one
+# runner disk is what exhausted it in run 33295159787.
+#
+# Exit status: 0 on success, non-zero if the manifest or blob could not be
+# fetched or the extraction failed. Callers decide whether that is fatal.
+set -euo pipefail
+
+REF="${1:?usage: $0 <oras-ref> [<cache-dir>]}"
+CACHE_DIR="${2:-${HOME}/.cache/buildstream}"
+
+mkdir -p "$CACHE_DIR"
+
+if ! MANIFEST=$(oras manifest fetch "$REF" 2>&1); then
+    echo "cas-restore-stream: no manifest for ${REF}: ${MANIFEST}" >&2
+    exit 1
+fi
+
+DIGEST=$(printf '%s' "$MANIFEST" | jq -r '.layers[0].digest // empty')
+SIZE=$(printf '%s' "$MANIFEST" | jq -r '.layers[0].size // 0')
+if [ -z "$DIGEST" ]; then
+    echo "cas-restore-stream: ${REF} has no layers" >&2
+    exit 1
+fi
+
+# The ref may already carry a tag; oras accepts <repo>:<tag>@<digest>.
+echo "cas-restore-stream: streaming ${REF}@${DIGEST} ($((SIZE / 1024 / 1024)) MiB compressed) into ${CACHE_DIR}"
+# ghcr-blob-stream.py rather than `oras blob fetch --output -`: GHCR cuts
+# these multi-GB downloads with "PROTOCOL_ERROR" every few minutes and oras
+# cannot resume, so a cut threw the whole restore away. The helper resumes
+# with HTTP Range and verifies the digest.
+python3 "$(dirname "$0")/ghcr-blob-stream.py" "$REF" "$DIGEST" "$SIZE" \
+    | zstd -d -T0 \
+    | tar --exclude=sources -xf - -C "$CACHE_DIR"
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Stream one GHCR blob to stdout, resuming with HTTP Range after a cut.
+
+Usage: ghcr-blob-stream.py <ref> <digest> <size>
+  <ref>     e.g. ghcr.io/tuna-os/xfce-linux/cache-xfce-linux-core:latest
+  <digest>  sha256:... of the blob (from the manifest's layer)
+  <size>    blob size in bytes (from the manifest's layer)
+
+Why: `oras blob fetch --output -` has no resume. GHCR's blob downloads of
+the 10+ GB CAS tarballs get cut with "stream error: stream ID 1;
+PROTOCOL_ERROR" a few minutes in (runs 36973935390 and 36805045437, always
+on a 5-minute wall-clock boundary), and every cut threw the whole restore
+away, so jobs rebuilt from scratch. This reconnects at the byte offset it
+reached, and checks the sha256 of the full stream before exiting 0.
+
+Credentials come from the docker/oras auth file written by `oras login`
+(~/.docker/config.json), falling back to an anonymous token.
+"""
+
+import base64
+import hashlib
+import http.client
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+CHUNK = 4 * 1024 * 1024
+MAX_ATTEMPTS = 40
+
+
+def log(msg: str) -> None:
+    print(f"ghcr-blob-stream: {msg}", file=sys.stderr, flush=True)
+
+
+def basic_auth(registry: str) -> str | None:
+    for path in (
+        os.environ.get("DOCKER_CONFIG", "") and os.path.join(os.environ["DOCKER_CONFIG"], "config.json"),
+        os.path.expanduser("~/.docker/config.json"),
+        os.path.join(os.environ.get("XDG_RUNTIME_DIR", "/nonexistent"), "containers/auth.json"),
+    ):
+        if not path or not os.path.isfile(path):
+            continue
+        try:
+            auths = json.load(open(path)).get("auths", {})
+        except (OSError, ValueError):
+            continue
+        for key in (registry, f"https://{registry}"):
+            if auths.get(key, {}).get("auth"):
+                return auths[key]["auth"]
+    return None
+
+
+def bearer(registry: str, repo: str) -> str:
+    req = urllib.request.Request(f"https://{registry}/token?service={registry}&scope=repository:{repo}:pull")
+    auth = basic_auth(registry)
+    if auth:
+        req.add_header("Authorization", f"Basic {auth}")
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        return json.load(resp)["token"]
+
+
+def main() -> int:
+    if len(sys.argv) != 4:
+        print(__doc__, file=sys.stderr)
+        return 2
+    ref, digest, size = sys.argv[1], sys.argv[2], int(sys.argv[3])
+    registry, _, rest = ref.partition("/")
+    repo = rest.rsplit("@", 1)[0]
+    if ":" in repo.rsplit("/", 1)[-1]:
+        repo = repo.rsplit(":", 1)[0]
+    url = f"https://{registry}/v2/{repo}/blobs/{digest}"
+
+    out = sys.stdout.buffer
+    sha = hashlib.sha256()
+    offset = 0
+    attempt = 0
+    token = bearer(registry, repo)
+    while offset < size:
+        attempt += 1
+        if attempt > MAX_ATTEMPTS:
+            log(f"giving up after {MAX_ATTEMPTS} attempts at byte {offset}/{size}")
+            return 1
+        req = urllib.request.Request(url)
+        # Not forwarded on the redirect to the signed storage URL.
+        req.add_unredirected_header("Authorization", f"Bearer {token}")
+        if offset:
+            req.add_header("Range", f"bytes={offset}-")
+        try:
+            with urllib.request.urlopen(req, timeout=120) as resp:
+                if offset and resp.status != 206:
+                    log(f"server ignored Range (HTTP {resp.status}); cannot resume")
+                    return 1
+                while offset < size:
+                    buf = resp.read(min(CHUNK, size - offset))
+                    if not buf:
+                        break
+                    out.write(buf)
+                    sha.update(buf)
+                    offset += len(buf)
+        except urllib.error.HTTPError as exc:
+            if exc.code == 401:
+                token = bearer(registry, repo)
+            log(f"HTTP {exc.code} at byte {offset}/{size} (attempt {attempt}); retrying")
+            time.sleep(min(30, 2 * attempt))
+            continue
+        except (OSError, ValueError, http.client.HTTPException) as exc:
+            log(f"connection cut at byte {offset}/{size} (attempt {attempt}): {exc}; resuming")
+            time.sleep(min(30, 2 * attempt))
+            continue
+        if offset < size:
+            log(f"short read at byte {offset}/{size} (attempt {attempt}); resuming")
+    out.flush()
+    if offset != size:
+        log(f"read {offset} bytes, expected {size}")
+        return 1
+    got = "sha256:" + sha.hexdigest()
+    if got != digest:
+        log(f"digest mismatch: got {got}, expected {digest}")
+        return 1
+    log(f"done: {size} bytes in {attempt} attempt(s), digest verified")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())