Skip to content

Commit aa326d2

Browse files
committed
ci(refactor): port gguf publish to bash script
Move the entire GGUF conversion + publish flow into scripts/publish-gguf.sh so the workflow stays a thin shell: - variant short-name -> HF checkpoint id mapping lives in the script (case statement), not duplicated in the workflow matrix - venv at .venv-publish/ is created once and reused across matrix legs (saves reinstalling torch/transformers 8x per run) - SKIP_UPLOAD=1 runs the conversion + loadable-tensor check without touching the HF API; used by the validate job - HF_TOKEN guard is in the script (not silently omitted in workflow env) - GGUF loadability check moved into the script too, so it runs on every matrix leg not just validate The validate job is now a single bash invocation, and each publish matrix leg is also a single bash invocation. No more inline 'uv run python' / 'hf' / 'uv tool install' calls in the workflow. Script passes bash -n and shellcheck clean. Verified the pyproject.toml has no [gguf] extra, so the venv installs the project editable + only the one missing runtime dep (huggingface_hub[hf_transfer]).
1 parent 195ec1b commit aa326d2

2 files changed

Lines changed: 118 additions & 58 deletions

File tree

‎.github/workflows/convert-and-publish-gguf.yml‎

Lines changed: 6 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ concurrency:
2020
cancel-in-progress: false
2121

2222
env:
23-
UV_VERSION: "0.9.x"
23+
HF_HUB_ENABLE_HF_TRANSFER: "1"
2424

2525
jobs:
2626
validate:
@@ -31,26 +31,11 @@ jobs:
3131
- uses: actions/checkout@v4
3232

3333
- uses: astral-sh/setup-uv@v5
34-
with:
35-
version: ${{ env.UV_VERSION }}
3634

37-
- name: Install dependencies
38-
run: uv sync
35+
- name: Convert dinov2-small (dry run, no upload)
36+
run: SKIP_UPLOAD=1 bash scripts/publish-gguf.sh dinov2-small
3937

40-
- name: Convert dinov2-small (dry run)
41-
run: uv run python scripts/dinov2-to-gguf.py --model_name facebook/dinov2-small-imagenet1k-1-layer
42-
43-
- name: Verify GGUF is loadable
44-
run: |
45-
uv run python - <<'EOF'
46-
import gguf
47-
reader = gguf.GGUFReader("./ggml-model.gguf")
48-
tensors = [t.name for t in reader.tensors]
49-
assert len(tensors) > 0, "no tensors in GGUF"
50-
print(f"OK: {len(tensors)} tensors")
51-
EOF
52-
53-
convert:
38+
publish:
5439
name: Convert ${{ matrix.variant }}
5540
needs: validate
5641
if: ${{ !cancelled() && needs.validate.result == 'success' }}
@@ -61,24 +46,15 @@ jobs:
6146
matrix:
6247
include:
6348
- variant: dinov2-small
64-
hf_id: facebook/dinov2-small-imagenet1k-1-layer
6549
- variant: dinov2-base
66-
hf_id: facebook/dinov2-base-imagenet1k-1-layer
6750
- variant: dinov2-large
68-
hf_id: facebook/dinov2-large-imagenet1k-1-layer
6951
- variant: dinov2-giant
70-
hf_id: facebook/dinov2-giant-imagenet1k-1-layer
7152
- variant: dinov2-with-registers-small
72-
hf_id: facebook/dinov2-with-registers-small-imagenet1k-1-layer
7353
- variant: dinov2-with-registers-base
74-
hf_id: facebook/dinov2-with-registers-base-imagenet1k-1-layer
7554
- variant: dinov2-with-registers-large
76-
hf_id: facebook/dinov2-with-registers-large-imagenet1k-1-layer
7755
- variant: dinov2-with-registers-giant
78-
hf_id: facebook/dinov2-with-registers-giant-imagenet1k-1-layer
7956
env:
8057
HF_TOKEN: ${{ secrets.HF_TOKEN }}
81-
HF_HUB_ENABLE_HF_TRANSFER: "1"
8258
steps:
8359
- name: Skip unselected variants (manual dispatch)
8460
if: ${{ github.event_name == 'workflow_dispatch' && inputs.variant != 'all' && inputs.variant != matrix.variant }}
@@ -89,35 +65,7 @@ jobs:
8965

9066
- uses: astral-sh/setup-uv@v5
9167
if: env.SKIP != 'true'
92-
with:
93-
version: ${{ env.UV_VERSION }}
94-
95-
- name: Free disk space (giant variants)
96-
if: env.SKIP != 'true' && contains(matrix.variant, 'giant')
97-
uses: jlumbroso/free-disk-space@main
98-
with:
99-
tool-cache: false
100-
android: true
101-
dotnet: true
102-
haskell: true
103-
large-packages: true
104-
docker-images: true
105-
swap-storage: false
106-
107-
- name: Install dependencies
108-
if: env.SKIP != 'true'
109-
run: |
110-
uv sync
111-
uv tool install 'huggingface_hub[cli]'
112-
113-
- name: Convert to GGUF
114-
if: env.SKIP != 'true'
115-
run: uv run python scripts/dinov2-to-gguf.py --model_name ${{ matrix.hf_id }}
116-
117-
- name: Ensure target repo exists
118-
if: env.SKIP != 'true'
119-
run: hf repo create dinov2-cpp-core/${{ matrix.variant }}-gguf --repo_type model --exist_ok
12068

121-
- name: Upload GGUF to Hugging Face
69+
- name: Convert and publish ${{ matrix.variant }}
12270
if: env.SKIP != 'true'
123-
run: hf upload dinov2-cpp-core/${{ matrix.variant }}-gguf ./ggml-model.gguf model.gguf
71+
run: bash scripts/publish-gguf.sh "${{ matrix.variant }}"

‎scripts/publish-gguf.sh‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#!/usr/bin/env bash
2+
# Convert a DINOv2 HuggingFace checkpoint to GGUF and optionally publish to the
3+
# `dinov2-cpp-core` HuggingFace organisation.
4+
#
5+
# Usage:
6+
# bash scripts/publish-gguf.sh <variant-short-name>
7+
#
8+
# Environment:
9+
# HF_TOKEN Required unless SKIP_UPLOAD=1. Token used by `hf` CLI for
10+
# repo creation and upload.
11+
# SKIP_UPLOAD If set to 1, runs the conversion but skips `hf repo create`
12+
# and `hf upload`. Used by CI's validate job.
13+
#
14+
# The script creates a project-local venv at .venv-publish/ (reused across runs
15+
# to avoid reinstalling torch/transformers on every matrix leg), installs the
16+
# project editable plus the HuggingFace CLI, converts the model, and uploads
17+
# the resulting ggml-model.gguf to dinov2-cpp-core/<variant>-gguf.
18+
19+
set -euo pipefail
20+
21+
if [ "$#" -ne 1 ]; then
22+
echo "usage: $0 <variant-short-name>" >&2
23+
echo " e.g. $0 dinov2-small" >&2
24+
exit 2
25+
fi
26+
27+
VARIANT="$1"
28+
29+
# Map variant short-name -> facebookresearch HF checkpoint id.
30+
case "$VARIANT" in
31+
dinov2-small) HF_ID="facebook/dinov2-small-imagenet1k-1-layer" ;;
32+
dinov2-base) HF_ID="facebook/dinov2-base-imagenet1k-1-layer" ;;
33+
dinov2-large) HF_ID="facebook/dinov2-large-imagenet1k-1-layer" ;;
34+
dinov2-giant) HF_ID="facebook/dinov2-giant-imagenet1k-1-layer" ;;
35+
dinov2-with-registers-small) HF_ID="facebook/dinov2-with-registers-small-imagenet1k-1-layer" ;;
36+
dinov2-with-registers-base) HF_ID="facebook/dinov2-with-registers-base-imagenet1k-1-layer" ;;
37+
dinov2-with-registers-large) HF_ID="facebook/dinov2-with-registers-large-imagenet1k-1-layer" ;;
38+
dinov2-with-registers-giant) HF_ID="facebook/dinov2-with-registers-giant-imagenet1k-1-layer" ;;
39+
*)
40+
echo "ERROR: unknown variant '$VARIANT'" >&2
41+
echo " known variants: dinov2-{small,base,large,giant} dinov2-with-registers-{small,base,large,giant}" >&2
42+
exit 2
43+
;;
44+
esac
45+
46+
REPO_SLUG="dinov2-cpp-core/${VARIANT}-gguf"
47+
GGUF_PATH="./ggml-model.gguf"
48+
VENV_DIR=".venv-publish"
49+
50+
# ---- HF_TOKEN guard (only when we actually need to upload) -----------------
51+
if [ "${SKIP_UPLOAD:-0}" != "1" ] && [ -z "${HF_TOKEN:-}" ]; then
52+
echo "ERROR: HF_TOKEN is required (set it in the environment or use SKIP_UPLOAD=1)" >&2
53+
exit 1
54+
fi
55+
56+
# ---- venv setup ------------------------------------------------------------
57+
if [ ! -d "$VENV_DIR" ]; then
58+
echo "==> creating venv at $VENV_DIR"
59+
uv venv "$VENV_DIR"
60+
# shellcheck disable=SC1091
61+
source "$VENV_DIR/bin/activate"
62+
echo "==> installing project (editable) + huggingface_hub[hf_transfer]"
63+
# -e . pulls in all [project] dependencies (torch, transformers, gguf, ...).
64+
# huggingface_hub is the only runtime dep not declared in pyproject.toml;
65+
# [hf_transfer] pulls in hf_transfer for HF_HUB_ENABLE_HF_TRANSFER=1.
66+
uv pip install -e . 'huggingface_hub[hf_transfer]'
67+
else
68+
echo "==> reusing venv at $VENV_DIR"
69+
# shellcheck disable=SC1091
70+
source "$VENV_DIR/bin/activate"
71+
fi
72+
73+
# Make HF transfer explicit inside the script as well — protects against the
74+
# caller forgetting to export it.
75+
export HF_HUB_ENABLE_HF_TRANSFER="${HF_HUB_ENABLE_HF_TRANSFER:-1}"
76+
77+
# ---- conversion ------------------------------------------------------------
78+
echo "==> converting $VARIANT ($HF_ID) -> GGUF"
79+
python scripts/dinov2-to-gguf.py --model_name "$HF_ID"
80+
81+
if [ ! -s "$GGUF_PATH" ]; then
82+
echo "ERROR: $GGUF_PATH missing or empty after conversion" >&2
83+
exit 1
84+
fi
85+
SIZE_BYTES=$(stat -c%s "$GGUF_PATH" 2>/dev/null || stat -f%z "$GGUF_PATH")
86+
echo "==> wrote $GGUF_PATH ($SIZE_BYTES bytes)"
87+
88+
# ---- verify the GGUF is loadable ------------------------------------------
89+
echo "==> verifying GGUF is loadable"
90+
python - <<'PYEOF'
91+
import gguf
92+
reader = gguf.GGUFReader("./ggml-model.gguf")
93+
tensors = [t.name for t in reader.tensors]
94+
assert len(tensors) > 0, "no tensors in GGUF"
95+
print(f"OK: {len(tensors)} tensors")
96+
PYEOF
97+
98+
# ---- upload ----------------------------------------------------------------
99+
if [ "${SKIP_UPLOAD:-0}" = "1" ]; then
100+
echo "==> SKIP_UPLOAD=1, not uploading to $REPO_SLUG"
101+
else
102+
echo "==> ensuring HF repo $REPO_SLUG exists"
103+
hf repo create "$REPO_SLUG" --repo_type model --exist_ok
104+
105+
echo "==> uploading $GGUF_PATH -> $REPO_SLUG (path: model.gguf)"
106+
hf upload "$REPO_SLUG" "$GGUF_PATH" model.gguf
107+
fi
108+
109+
# ---- cleanup ---------------------------------------------------------------
110+
# Remove the GGUF artifact; keep the venv so the next matrix leg can reuse it.
111+
rm -f "$GGUF_PATH"
112+
echo "==> done: $VARIANT"

0 commit comments

Comments
 (0)