Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions scripts/build_logo_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,15 +75,24 @@ def assign(existing: dict[str, str], keys: list[str], prefix: str) -> None:
model_keys = [f"{model}␟{organization}" for model, organization in models]
from benchmark_radar.models_registry import model_key

by_identity = {}
# A renamed model inherits its old label's ID, but only from a label the
# data no longer carries. models.json can keep two live records whose
# slugs collide ("Gemini 2.5 Flash" and "Gemini-2.5-Flash"); letting the
# new one inherit from a still-live twin handed both cards the same ID.
live_labels = set(model_keys)
by_identity: dict[str, list[str]] = {}
for label, identifier in model_ids.items():
if label in live_labels:
continue
model, organization = label.split("␟", 1)
by_identity.setdefault(model_key(model, organization), identifier)
by_identity.setdefault(model_key(model, organization), []).append(identifier)
for label in model_keys:
if label in model_ids:
continue
model, organization = label.split("␟", 1)
previous = by_identity.get(model_key(model, organization))
if label not in model_ids and previous:
model_ids[label] = previous
donors = by_identity.get(model_key(model, organization))
if donors:
model_ids[label] = donors.pop(0)
assign(model_ids, model_keys, "M")

# An entry the data no longer carries is dropped. Freezing an ID protects a
Expand Down
44 changes: 44 additions & 0 deletions tests/test_models_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,50 @@ def test_the_logo_generator_preserves_a_retired_high_water_mark(tmp_path, monkey
assert generated["high_water"] == {"O": 68, "M": 1048}


def test_slug_twins_that_are_both_live_get_distinct_logo_ids(tmp_path, monkeypatch):
"""Rename inheritance must not hand a live label's ID to its slug twin.

models.json keeps "Gemini 2.5 Flash" and "Gemini-2.5-Flash" as separate
records; inheriting from the still-live twin gave 36 cards a shared ID.
A retired label's ID is still inherited by its renamed successor.
"""
script = Path("scripts/build_logo_registry.py").resolve()
data_dir = tmp_path / "site" / "data"
data_dir.mkdir(parents=True)
(data_dir / "models.json").write_text(
json.dumps(
{
"models": [
{"model": "Gemini 2.5 Flash", "organization": "Google"},
{"model": "Gemini-2.5-Flash", "organization": "Google"},
{"model": "Grok 4", "organization": "xAI"},
]
}
),
encoding="utf-8",
)
(data_dir / "logo-registry.json").write_text(
json.dumps(
{
"high_water": {"O": 2, "M": 2},
"organizations": {"Google": "O-01", "xAI": "O-02"},
"models": {"Gemini 2.5 Flash␟Google": "M-01", "Grok-4␟xAI": "M-02"},
}
),
encoding="utf-8",
)

monkeypatch.chdir(tmp_path)
runpy.run_path(str(script), run_name="__main__")

models = json.loads((data_dir / "logo-registry.json").read_text(encoding="utf-8"))["models"]
assert models == {
"Gemini 2.5 Flash␟Google": "M-01",
"Grok 4␟xAI": "M-02",
"Gemini-2.5-Flash␟Google": "M-03",
}


def test_a_missing_shard_directory_refuses_to_write_a_curated_only_registry(tmp_path):
"""The 321-model drop this module opens on, reachable again since the
shards stopped being committed.
Expand Down
Loading