Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
bf5cc54
docs: specify parity benchmark and binary preview
espetro Sep 21, 2026
312ea0e
fix(cli): enforce strict numeric parsing
espetro Sep 21, 2026
749e67e
fix(cli): address Task 1 review nits
espetro Sep 21, 2026
2d75afe
feat(cli): add preview binary embeddings output
espetro Sep 21, 2026
dd33647
fix(cli): address binary embeddings review findings
espetro Sep 21, 2026
a9e41e6
docs: add reproducible parity evidence
espetro Sep 21, 2026
6547bd7
docs(parity): clarify evidence provenance and downloads
espetro Sep 21, 2026
c48472c
docs(bench): report blocked Ubuntu evidence run
espetro Sep 21, 2026
b0359a5
fix(ci): use supported Hugging Face CLI
espetro Sep 21, 2026
b31ca0b
docs: record Ubuntu benchmark rerun blocker
espetro Sep 21, 2026
50d5dee
fix(bench): parse benchmark JSON portably
espetro Sep 21, 2026
b62ca23
docs(bench): record Ubuntu benchmark artifact
espetro Sep 21, 2026
9d0a626
ci(bench): preserve model provenance
espetro Sep 21, 2026
4db55b3
docs(bench): record current Ubuntu benchmark evidence
espetro Sep 21, 2026
d951fb7
feat: harden backbone-only DINOv2 loading
espetro Sep 21, 2026
6367a97
test: harden backbone loader review coverage
espetro Sep 21, 2026
e703c19
ci(bench): publish Ubuntu-only benchmark evidence
espetro Sep 21, 2026
7bd6fa4
fix: address final parity review findings
espetro Sep 21, 2026
a317259
docs: refresh final parity evidence
espetro Sep 21, 2026
c9e5a2a
fix: support backbone-only DINOv2 conversion
espetro Sep 21, 2026
e04b70e
fix(converter): harden DINOv2 backbone resolution
espetro Sep 21, 2026
100efb2
docs: refresh final parity evidence
espetro Sep 21, 2026
db0d72d
docs: fix regular GGUF checksum
espetro Sep 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 51 additions & 12 deletions .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,19 @@ on:
workflow_dispatch:
inputs:
variant:
description: "Which f16 GGUF to bench (single variant or 'all')"
description: "Which published f16 GGUF to bench (single variant or 'all')"
required: false
default: "small"
type: choice
options:
- small
- small-registers
- base
- base-registers
- large
- large-registers
- giant
- giant-registers
- all
repeats:
description: "Timed runs per cell (forwarded to scripts/bench.sh --repeats)"
Expand Down Expand Up @@ -90,19 +94,29 @@ jobs:
BENCH_MODELS: ${{ steps.variants.outputs.BENCH_MODELS }}
run: |
set -euo pipefail
# scripts/bench.sh hard-codes paths of the form
# `models/dinov2-vit-<variant>-patch14/model.gguf`. The HF
# repos ship `dinov2-{small,base,large,giant}-gguf` and the
# uploaded file is named `model.gguf` (no quant suffix); f16
# is the only quant this repo benches natively per the v0.3.0
# commit that dropped the --quants loop.
# scripts/bench.sh uses a separate directory for each published
# regular or register-token checkpoint. Each HF repo uploads its
# f16 file as model.gguf; no quantization benchmark selector is used.
variant_list="${BENCH_MODELS//,/ }"
for variant in $variant_list; do
repo="dinov2-cpp-core/dinov2-${variant}-gguf"
target_dir="models/dinov2-vit-${variant}-patch14"
case "$variant" in
small|base|large|giant)
repo="dinov2-cpp-core/dinov2-${variant}-gguf"
target_dir="models/dinov2-vit-${variant}-patch14"
;;
small-registers|base-registers|large-registers|giant-registers)
size="${variant%-registers}"
repo="dinov2-cpp-core/dinov2-with-registers-${size}-gguf"
target_dir="models/dinov2-with-registers-${size}-patch14"
;;
*)
echo "::error::Unsupported benchmark variant: $variant" >&2
exit 1
;;
esac
mkdir -p "$target_dir"
echo "Downloading $repo -> $target_dir/model.gguf"
huggingface-cli download \
hf download \
--repo-type model \
--local-dir "$target_dir" \
"$repo" "model.gguf"
Expand All @@ -111,23 +125,46 @@ jobs:
- name: Verify staged weights
env:
BENCH_MODELS: ${{ steps.variants.outputs.BENCH_MODELS }}
BENCH_VARIANT_INPUT: ${{ inputs.variant }}
run: |
set -euo pipefail
provenance_path=benchmark_provenance.txt
{
printf 'workflow=bench.yml\n'
printf 'run_id=%s\n' "$GITHUB_RUN_ID"
printf 'run_attempt=%s\n' "$GITHUB_RUN_ATTEMPT"
printf 'commit=%s\n' "$GITHUB_SHA"
printf 'variant_input=%s\n' "$BENCH_VARIANT_INPUT"
printf 'models=%s\n' "$BENCH_MODELS"
printf 'image_path=assets/tench.jpg\n'
printf 'build_configuration=Release\n'
printf 'platform=ubuntu-latest\n'
printf 'quantization=f16 only\n'
} > "$provenance_path"
variant_list="${BENCH_MODELS//,/ }"
for variant in $variant_list; do
target="models/dinov2-vit-${variant}-patch14/model.gguf"
case "$variant" in
small|base|large|giant) target="models/dinov2-vit-${variant}-patch14/model.gguf" ;;
small-registers|base-registers|large-registers|giant-registers) target="models/dinov2-with-registers-${variant%-registers}-patch14/model.gguf" ;;
*) echo "::error::Unsupported benchmark variant: $variant" >&2; exit 1 ;;
esac
if [ ! -f "$target" ]; then
echo "::error::Missing $target after download" >&2
exit 3
fi
printf "%-50s %10d bytes\n" "$target" "$(stat -c %s "$target")"
printf 'model_path=%s\n' "$target" >> "$provenance_path"
sha256sum "$target" >> "$provenance_path"
done
echo "--- $provenance_path ---"
cat "$provenance_path"

- name: Run benchmarks
env:
BENCH_MODELS: ${{ steps.variants.outputs.BENCH_MODELS }}
BENCH_REPEATS: ${{ inputs.repeats }}
BENCH_THREADS: ${{ inputs.threads }}
BENCH_BUILD_CONFIG: Release
run: |
set -euo pipefail
echo "models selected: $BENCH_MODELS"
Expand All @@ -148,6 +185,8 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: bench-results-${{ inputs.variant }}-${{ github.run_id }}
path: benchmark_results.txt
path: |
benchmark_results.txt
benchmark_provenance.txt
if-no-files-found: error
retention-days: 90
71 changes: 71 additions & 0 deletions .superpowers/sdd/2026-09-21-parity-bench-binary/task-4-report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# Task 4 report: Ubuntu benchmark evidence

## Status

Completed for the feasible Ubuntu small regular model. The committed result is
an actual successful GitHub Actions measurement, not seed data. The benchmark
workflow and script now record provenance and support explicitly named regular
and register-token f16 variants. No models, binaries, build outputs, release
artifacts, or quantization benchmark data were committed.

## Changes

- `.github/workflows/bench.yml` remains Ubuntu-only and CPU-only. Its input
mapping now supports `small-registers`, `base-registers`,
`large-registers`, and `giant-registers` in addition to regular variants.
`all` intentionally remains the four regular sizes to avoid silently
expanding runner storage and runtime.
- `scripts/bench.sh` now validates the named variants and writes runner, git
SHA, precision, backend, batch, command, exact GGUF path, and GGUF SHA-256
alongside the timing columns.
- `benchmark_results.txt` replaces the placeholder table with the successful
Ubuntu result below.
- `docs/benchmarks.md` labels current evidence Ubuntu-only, documents
reproduction and limitations, and separates historical comparison data.

## Actual benchmark evidence

- Workflow: `bench.yml`, run [35630688827](https://github.com/espetro/dinov2.cpp/actions/runs/35630688827)
- Workflow result: success
- Generated: `2026-09-21T17:14:58Z`
- Runner: `ubuntu-latest`, reported as `linux-x86_64`
- Run commit: `9d0a6265ae53088e423e4c2417ec8589de181e99`
- Model: regular small, `dinov2-cpp-core/dinov2-small-gguf`
- GGUF path: `models/dinov2-vit-small-patch14/model.gguf`
- GGUF SHA-256: `b7ca009aa416f6be85ea95363f4ef13d3c139999f96ca3fd47226849afb712e`
- Precision/backend: f16 GGUF, CPU
- Image: `assets/tench.jpg`; batch: 1
- Threads: 12; warmup: 1; timed runs: 5
- Command: `scripts/bench.sh --models small --repeats 5 --threads 12 --image assets/tench.jpg --bin build/bin/dinov2-cli --out benchmark_results.txt`
- Measurement: mean `255.0 ms`, stddev `9.1 ms`, min `249 ms`, max `271 ms`, peak RSS `104 MB`

The result and the downloaded workflow provenance artifact were inspected with:

```text
gh run download 35630688827 --name bench-results-small-35630688827 --dir /tmp/bench-run-35630688827
cat /tmp/bench-run-35630688827/benchmark_results.txt
cat /tmp/bench-run-35630688827/benchmark_provenance.txt
```

## Coverage and limitations

Measured coverage is one regular small f16 GGUF on Ubuntu CPU only. Register-token
variants and base, large, and giant sizes remain unmeasured in the committed
result. `all` is not claimed as measured. macOS, Windows, ARM, GPU, and
quantization-specific comparisons remain out of scope.

The successful artifact was produced at run SHA
`9d0a6265ae53088e423e4c2417ec8589de181e99`, while the current local branch is
`6547bd7bfbe39232cf568215db6262d6178020fa`. The local branch was already ahead
of its published remote branch, and no push was performed. Therefore the
artifact is evidence for the benchmark behavior and model command represented
by that successful workflow run, not a claim that the newly edited workflow
mapping itself has executed. A fresh run of the modified workflow requires
publishing the commit, which was intentionally not done for this task.

## Verification

- `bash -n scripts/bench.sh` and `scripts/bench.sh --help`: pass.
- Static workflow token checks: pass.
- `actionlint .github/workflows/bench.yml`: pass.
- `git diff --check`: pass.
78 changes: 78 additions & 0 deletions .superpowers/sdd/2026-09-21-parity-bench-binary/task-5-report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Task 5 report: model coverage assessment and scoped backbone support

Date: 2026-09-21
Branch: `feat/batched-inference`

## Status

PASS. Backbone-only DINOv2 feature mode is a small, semantically safe extension of the existing graph. It does not add a new architecture or task head. The loader now tolerates omitted optional register-token metadata and rejects classification requests before any classifier tensor access when the GGUF is backbone-only or incomplete.

DINOv3, depth, segmentation, and other task heads remain out of scope. No weights were downloaded or committed, and no publish workflow was expanded for new repositories.

## Coverage matrix

| Checkpoint family | Converter | Loader feature mode | Loader classification | Publish mapping | Evidence |
|---|---|---|---|---|---|
| `facebook/dinov2-{small,base,large,giant}` | Supported by the existing `AutoModel` path and common tensor conversion | Supported for GGUFs with the DINOv2 backbone tensors; missing `num_register_tokens` defaults to zero | Unsupported by design because no ImageNet head is present; now a clear error | Not mapped | No model download or parity claim in this task |
| `facebook/dinov2-with-registers-{small,base,large,giant}` | Existing register tensor and layer selection path | Supported when register metadata and tensor are present | Unsupported unless a classifier head and labels are present | Not mapped as backbone-only variants | Existing small register classifier parity remains in Task 3 |
| `facebook/dinov2-{size}-imagenet1k-1-layer` | Existing classifier conversion path | Supported | Supported when classifier tensors, `num_classes`, and labels are present | Mapped in the eight-variant publish workflow | Existing small regular parity evidence only |
| `facebook/dinov2-with-registers-{size}-imagenet1k-1-layer` | Existing classifier conversion path | Supported | Supported when classifier tensors, `num_classes`, and labels are present | Mapped in the eight-variant publish workflow | Existing small register parity evidence only |
| DINOv2 depth or segmentation heads | Not implemented | Not applicable | Not applicable | Not mapped | Explicitly deferred |
| DINOv3 | Not implemented | Not applicable | Not applicable | Not mapped | Separate architecture and resource target; explicitly deferred |

## Evidence and behavior inspection

* `scripts/dinov2-to-gguf.py:42-50` selects `AutoModelForImageClassification` only for model names containing `imagenet`; other names use `AutoModel`. The converter writes `num_classes: 0` and no labels for a backbone-only checkpoint, while the shared encoder and embedding tensors remain the same. It currently writes `num_register_tokens: 0` at line 124, but the loader should not require that optional key from independently produced GGUFs.
* `dinov2.cpp:255-272` loads the required backbone hyperparameters and now uses a safe zero default for absent `num_register_tokens`.
* `dinov2.cpp:285-318` validates classification metadata, labels, and `classifier.weight` / `classifier.bias` before graph construction. A backbone-only request with `-c` therefore returns a loader error rather than reaching the `model.tensors.at(...)` calls in `forward_head` at `dinov2.cpp:693-694`.
* Feature mode does not call `forward_head`. `build_graph` at `dinov2.cpp:705-715` only adds the head when `params.classify` is true. `forward_features` always emits CLS and non-register patch tokens, and the feature output path builds pooled vectors from those tokens. This gives backbone-only models the existing CLS, pooled, patch-token, and PCA semantics.
* `dinov2.h:106-112` defines the output contract: CLS in both modes, pooled and patch tokens in feature mode, and classification predictions only in classify mode.
* `scripts/publish-gguf.sh:56-71` maps only the eight ImageNet classifier checkpoint names. `.github/workflows/convert-and-publish-gguf.yml:38-46` repeats that exact eight-variant matrix. `docs/hf-publishing.md:32-43` documents the same eight published repositories. Since the repository does not already intend to publish backbone-only repositories, this task leaves those mappings unchanged.
* `docs/cli.md:121-139` now documents the supported feature-only backbone coverage, the clean classification failure, the zero-register default, the publish limitation, and the explicit DINOv3 and other-head boundaries.

## Changes

* `dinov2.cpp`: added optional GGUF uint32 metadata lookup; defaulted absent `num_register_tokens` to zero; validated classifier metadata, label keys, and tensors before classification loading.
* `tests/test_cli.cpp`: added a no-download malformed-model classification smoke test that asserts exit 1, empty stdout, a loader failure, and no assertion text. Existing parser, help, and version cases remain unchanged.
* `docs/cli.md`: added the coverage and scope decision.
* No converter, publish mapping, or weights were changed because the existing converter and eight-repository workflow do not publish backbone-only models.

## Verification

* `clang-format -i dinov2.cpp tests/test_cli.cpp`: PASS.
* `cmake --build --preset debug -j2`: PASS.
* `cmake --preset debug -DDINOV2_FATAL_WARNINGS=ON && cmake --build --preset debug --clean-first -j2`: PASS.
* `ctest --test-dir build-debug --output-on-failure`: PASS, 3/3 tests. The focused CLI test took 20.87 seconds because each subprocess initializes the CLI backend.
* `git diff --check`: PASS.

## Follow-up recommendations

1. If backbone-only weights become a publishing goal, add explicitly named HF mappings and separate conversion validation plus parity artifacts for each model size. Do not infer base, large, or giant parity from the small checkpoint.
2. Add a real small synthetic backbone-only GGUF fixture to exercise the successful loader path and omitted metadata path in CI. The current repository tests avoid model fixtures and this task did not download a giant or other model.
3. Treat task heads and DINOv3 as separate design and compatibility work rather than extending the current classifier graph.

## Review fix notes

* Replaced the malformed-file CLI smoke test with a deterministic, valid GGUF generated in `tests/test_cli.cpp`. The fixture contains only the required backbone metadata, omits `num_classes` and classifier tensors, and verifies `-c` fails at the classification preflight with no assertion text. The same assertion checks the omitted `num_register_tokens` path reports a zero default.
* Added a second minimal GGUF fixture containing `embeddings.register_tokens` without `num_register_tokens`. The loader now rejects this inconsistency before model tensors are loaded, preventing the tensor from being silently treated as zero registers. It also rejects the inverse mismatch while preserving models with neither metadata nor tensor.
* Re-ran clang-format 18.1.8, the debug build, focused `test_cli`, full ctest, and `git diff --check`.

## Consolidated final review fix wave: 2026-09-21

This fix wave replaces actionable `huggingface-cli download` instructions with `hf download` while preserving repositories, paths, and arguments. Benchmark generation now records the image path `assets/tench.jpg`, `Release` build configuration, Ubuntu platform scope, f16-only quantization, model paths, checksums, commit, and run metadata in generated evidence and workflow provenance. The committed result retains the canonical successful Ubuntu run; earlier failed or parser-debug attempts are explicitly historical in the task-4 report.

The loader now validates positive register-token metadata against `embeddings.register_tokens` dimensions before graph execution: hidden size, register count, and singleton trailing dimensions must match. No-register and valid backbone-only models remain supported. CLI tests cover hidden, count, and trailing-dimension mismatches. New CLI tests use a unique directory under `std::filesystem::temp_directory_path()` and remove it through an RAII cleanup guard. Classification now rejects `--topk` greater than model `num_classes` after model load with exit status 1 and an actionable error; the focused fixture covers this behavior.

Verification performed for this wave: debug build, CTest, CLI-focused fixture tests, benchmark parser regression, actionlint, clang-format 18, and `git diff --check`. No release or merge was run, and no push was performed.

## Converter defect fix: 2026-09-21

The earlier claim that no converter changes were needed was incorrect for public backbone-only checkpoints. `scripts/dinov2-to-gguf.py` now resolves the underlying DINOv2 module structurally through the model's `base_model` and module tree. This supports both direct `AutoModel` backbones and `AutoModelForImageClassification` wrappers without assuming the version-specific `model.dinov2` or `model.dinov2_with_registers` attributes. Encoder QKV extraction uses the resolved module's `encoder.layer`.

Classifier conversion remains selected only for the existing ImageNet checkpoint naming convention. Classifier labels and tensors continue to be emitted only from that classifier path; backbone-only conversion writes no invented head or labels and remains feature-mode compatible. `CONTRIBUTING.md` now names the supported public backbone-only model families. `tests/test_dinov2_converter.py` provides a lightweight synthetic direct-backbone, wrapped-classifier, and rejection smoke test.

Verification for this fix: `python -m py_compile scripts/dinov2-to-gguf.py tests/test_dinov2_converter.py` passed. `python -m unittest -v tests/test_dinov2_converter.py` ran two tests and skipped both because `torch` is not installed. No public weights were downloaded, so a real conversion was not run and no GGUF was created. C++ sources were unchanged; `cmake --build --preset debug -j2` reported no work and `ctest --test-dir build-debug --output-on-failure` passed all 3 tests. `git diff --check` passed. No merge, release, or push was performed.

## Converter hardening fix note: 2026-09-21

The resolver now treats objects that do not support `vars()` as leaves instead of allowing its `TypeError` to escape. Backbone shape matching also checks an available `config.model_type`, accepting DINOv2 (including register variants) or config-less synthetic objects while rejecting generic ViT/DeiT-shaped modules. Tests cover both malformed slot-only inputs and a non-DINO config. No dependency setup or public weights were added; the converter test remains skipped when `torch`, Transformers, or gguf are unavailable.
2 changes: 1 addition & 1 deletion ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,7 @@ Deliberately untracked by `.gitignore` and not in release archives:
- `build/`, `build-*/`, `cmake-build-debug/` — local CMake output.
- `.venv/`, `.venv-publish/` — per-run converter venv (created by `publish-gguf.sh`).
- `*.gguf`, `/models`, `/data` — pre-converted weights live on HF; users pull
them with `huggingface-cli download`.
them with `hf download`.
- `.agents/` — agent working notes (memory + plans); force-added per global
AGENTS.md policy but not in release tarballs.
- `.publish-logs/` — per-run audit trail from `publish-gguf.sh`.
Expand Down
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,9 @@ if (DINOV2_BUILD_TESTS)
tests)
dinov2_target_flags(test_dinov2)
add_test(NAME test_dinov2 COMMAND test_dinov2)

add_executable(test_cli tests/test_cli.cpp)
target_link_libraries(test_cli PRIVATE ggml)
dinov2_target_flags(test_cli)
add_test(NAME test_cli COMMAND test_cli $<TARGET_FILE:dinov2-cli>)
endif ()
Loading
Loading