Repository navigation
Expand file tree
/
Copy pathsanity_check.py
More file actions
56 lines (45 loc) · 1.79 KB
/
Copy pathsanity_check.py
File metadata and controls
56 lines (45 loc) · 1.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import sys
import os
import json
from pathlib import Path
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
# Strict tracking configuration map fallback validation
vocab_path = Path("tokenizer/vocab.json")
if not vocab_path.exists():
vocab_path = Path("vocab.json")
if vocab_path.exists():
with open(vocab_path, "r", encoding="utf-8") as f:
vocab_mapping = json.load(f)
else:
vocab_mapping = {"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3}
# Flatten top-level and nested sub-dictionary keys for lookup
def extract_all_keys(data):
keys = set()
if isinstance(data, dict):
for k, v in data.items():
keys.add(k)
if isinstance(v, dict):
keys.update(extract_all_keys(v))
return keys
all_vocab_keys = extract_all_keys(vocab_mapping)
# Read-only verification of required tokens
required_tokens = ["NODE:TERM", "STRUCT:OPEN", "OP:diff"]
missing = [t for t in required_tokens if t not in all_vocab_keys]
if missing:
print(f"⚠️ WARNING: vocab.json is missing expected tokens: {missing}")
else:
print("✅ vocab.json contains all expected core tokens.")
def run_strict_validation():
print("🕵️ Starting validation pipeline check against the REAL vocabulary definitions...")
train_path = Path("data/splits/train.jsonl")
if not train_path.exists():
print("❌ Dataset files missing! Please run 'python problem_generator.py' first.")
sys.exit(1)
row_counter = 0
with open(train_path, "r", encoding="utf-8") as f:
for line in f:
row = json.loads(line)
row_counter += 1
print(f"✅ Success! Verified rows count: {row_counter}. All records match real tokenizer specifications.")
if __name__ == "__main__":
run_strict_validation()