{len(images)} images scanned, {len(groups)} cluster(s).
+{"".join(rows)}""" + path.write_text(doc, encoding="utf-8") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("cli", help="path to the dinov2-cli binary") + ap.add_argument("model", help="path to a GGUF model") + ap.add_argument("image_dir", type=Path, help="folder of images to scan") + ap.add_argument("-t", "--threshold", type=float, default=0.95, + help="cosine similarity cutoff for duplicates (default: 0.95)") + ap.add_argument("-o", "--out", type=Path, default=Path("dedup-report.html"), + help="HTML report path (default: dedup-report.html)") + args = ap.parse_args() + + images = sorted(p for p in args.image_dir.iterdir() if p.suffix.lower() in EXTS) + if len(images) < 2: + sys.exit(f"need at least 2 images in {args.image_dir}") + + print(f"embedding {len(images)} images...", file=sys.stderr) + records = embed(args.cli, args.model, images) + vecs = [r.get("pooled") or r["cls"] for r in records] + + sims = [(cosine(vecs[i], vecs[j]), i, j) + for i in range(len(images)) for j in range(i + 1, len(images))] + sims = sorted((s for s in sims if s[0] >= args.threshold), reverse=True) + + groups = clusters(sims, len(images)) + write_html(args.out, images, groups, sims) + for gi, group in enumerate(groups): + print(f"cluster {gi + 1}: " + ", ".join(images[i].name for i in group)) + print(f"wrote {args.out} ({len(groups)} cluster(s), threshold {args.threshold})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/image.cpp b/src/image.cpp index f29e5dd..643627d 100644 --- a/src/image.cpp +++ b/src/image.cpp @@ -9,6 +9,7 @@ constexpr float IMAGENET_DEFAULT_STD[3] = {0.229f, 0.224f, 0.225f}; #include "stb_image_write.h" #include