Skip to content

Commit b44635d

Browse files
authored
Merge pull request #43 from umer12x/DEV1_task
implementaion of dev1 task
2 parents cf95350 + 0a0115e commit b44635d

4 files changed

Lines changed: 359 additions & 108 deletions

File tree

‎inference/beam_search.py‎

Lines changed: 66 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,27 @@
2323
load_vocab,
2424
)
2525

26-
# NOTE: NodeValidityPool, flatten_vocab, load_vocab, is_valid_prefix and
27-
# is_complete live in inference/grammar.py (torch-free) so the ONNX
28-
# deployment path (deployment/onnx_beam_search.py) can reuse them without
29-
# importing torch.
30-
#
31-
# WARNING: deployment/onnx_beam_search.py mirrors this file's search loop.
32-
# The grammar-side termination fix reaches it automatically via the shared
33-
# import, but the loop changes here do NOT -- mirror them, or the ONNX path
34-
# keeps the never-terminating behaviour described below.
26+
def _call_model(
27+
model,
28+
src_tokens: torch.Tensor,
29+
tgt_tokens: torch.Tensor,
30+
src_positions: Optional[torch.Tensor] = None,
31+
parent_child_pairs: Optional[torch.Tensor] = None,
32+
) -> Any:
33+
try:
34+
return model(src_tokens, tgt_tokens)
35+
except TypeError:
36+
device = src_tokens.device
37+
batch_size, seq_len = src_tokens.size()
38+
if src_positions is None:
39+
src_positions = torch.zeros(
40+
(batch_size, seq_len, 3), dtype=torch.float32, device=device
41+
)
42+
if parent_child_pairs is None:
43+
parent_child_pairs = torch.zeros(
44+
(batch_size, seq_len, seq_len), dtype=torch.float32, device=device
45+
)
46+
return model(src_tokens, src_positions, parent_child_pairs, tgt_tokens)
3547

3648

3749
def _apply_repetition_penalty(
@@ -85,36 +97,6 @@ def beam_search(
8597
repetition_penalty: float = 1.2,
8698
repetition_min_count: int = 4,
8799
) -> Dict[str, Any]:
88-
"""Beam search for the tree-based CalculusSolverModel (model/transformer.py).
89-
90-
NOTE: CalculusSolverModel.forward(src_seq, tgt_in_seq, true_rule_ids=None)
91-
computes src_positions/parent_child_pairs internally (as zero tensors) and
92-
does not take them as inputs. src_positions/parent_child_pairs are accepted
93-
here only so callers built for the older tree-kwarg interface (e.g.
94-
inference/solve.py) don't break -- they are unused.
95-
96-
forward() returns (decoder_logits, rule_logits, verifier_logits); only
97-
decoder_logits is used for next-token scoring here. model_output is
98-
unpacked defensively (isinstance check) so this also works correctly
99-
if the model interface ever changes to a single-tensor return.
100-
101-
CORRECTION: an earlier version of this docstring attributed the 0%
102-
eval accuracy to exposure bias, and raised beam_size to compensate.
103-
That diagnosis was wrong and widening the beam could not have fixed it.
104-
The decoder had no legal way to terminate at any beam width: the grammar
105-
rejected every continuation of a closed AST, [EOS] included, so a beam
106-
that produced the correct answer had its entire candidate row masked to
107-
-inf and was dropped from the search. Only degenerate, still-open beams
108-
survived, which is why every prediction came back truncated and failed
109-
to deserialize. See inference/grammar.is_valid_prefix and
110-
tests/unit/test_beam_search.py::test_oracle_reaches_solved -- an oracle
111-
model that is certain about every token now recovers gold exactly, at
112-
every beam width from 1 to 8.
113-
114-
Beam width remains a real accuracy/CPU trade-off, but it should be
115-
re-tuned against a checkpoint now that termination works; the old
116-
0%/5.7% figures measured the termination bug, not the search width.
117-
"""
118100
device = src_tokens.device
119101
vocab = vocab_map["token_to_id"]
120102
id_to_token = vocab_map["id_to_token"]
@@ -138,27 +120,55 @@ def beam_search(
138120
vocab_size = max(id_to_token.keys()) + 1
139121
all_candidate_tokens = [id_to_token.get(idx, "[PAD]") for idx in range(vocab_size)]
140122

141-
beams = [{"tokens": [bos_id], "score": 0.0, "finished": False}]
123+
rule_token_entries = [
124+
tok for tok in vocab.keys() if tok.startswith("RULE:")
125+
]
126+
127+
seed_tokens = [bos_id]
128+
129+
if rule_token_entries:
130+
init_tgt = torch.tensor([[bos_id]], device=device)
131+
with torch.no_grad():
132+
init_output = _call_model(
133+
model,
134+
src_tokens,
135+
init_tgt,
136+
src_positions=src_positions,
137+
parent_child_pairs=parent_child_pairs,
138+
)
139+
init_rule_logits = init_output[1] if isinstance(init_output, tuple) else None
140+
141+
if init_rule_logits is not None:
142+
pred_rule_idx = torch.argmax(init_rule_logits, dim=-1).item()
143+
pred_rule_idx = min(pred_rule_idx, len(rule_token_entries) - 1)
144+
rule_token_str = rule_token_entries[pred_rule_idx]
145+
rule_token_id = vocab.get(rule_token_str)
146+
if rule_token_id is not None:
147+
seed_tokens = [bos_id, rule_token_id]
148+
149+
beams = [{"tokens": seed_tokens, "score": 0.0, "finished": False}]
142150
completed = []
143151

144152
for _ in range(max_len):
145153
candidates = []
146154
for beam in beams:
147155
current_tokens = beam["tokens"]
148-
token_strings = [id_to_token[t] for t in current_tokens]
149-
validity_tokens = (
150-
token_strings[1:]
151-
if token_strings and token_strings[0] == "[BOS]"
152-
else token_strings
153-
)
156+
token_strings = [id_to_token[t] for t in current_tokens if t in id_to_token]
157+
validity_tokens = token_strings[:]
158+
if validity_tokens and validity_tokens[0] == "[BOS]":
159+
validity_tokens = validity_tokens[1:]
160+
if validity_tokens and validity_tokens[0].startswith("RULE:"):
161+
validity_tokens = validity_tokens[1:]
154162

155163
tgt = torch.tensor([current_tokens], device=device)
156164

157-
# FIX: single model() call, unpacked defensively. A duplicate
158-
# second call to model() previously existed here (dead code
159-
# left over from a merge), doubling compute per step with no
160-
# behavioral difference -- removed.
161-
model_output = model(src_tokens, tgt)
165+
model_output = _call_model(
166+
model,
167+
src_tokens,
168+
tgt,
169+
src_positions=src_positions,
170+
parent_child_pairs=parent_child_pairs,
171+
)
162172
decoder_logits = model_output[0] if isinstance(model_output, tuple) else model_output
163173
next_logits = decoder_logits[0, -1, :]
164174

@@ -174,7 +184,11 @@ def beam_search(
174184
)
175185

176186
mask = node_pool.mask(validity_tokens, all_candidate_tokens)
177-
invalid_mask = torch.tensor([not v for v in mask], device=device)
187+
vocab_len = next_logits.size(0)
188+
padded_mask = mask + [True] * max(0, vocab_len - len(mask))
189+
invalid_mask = torch.tensor(
190+
[not v for v in padded_mask[:vocab_len]], device=device
191+
)
178192
safe_logits = next_logits.masked_fill(invalid_mask, float("-inf"))
179193

180194
# Hard repeat guard (task 2b). Applied as a mask, not a penalty:

‎inference/solve.py‎

Lines changed: 76 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,41 @@ class _LegacySLaNgTokenizer:
1515
pass
1616

1717

18+
class PklTransformerModel(torch.nn.Module):
19+
def __init__(self, vocab_size: int, hidden_dim: int = 128):
20+
super().__init__()
21+
self.pos_encoder = torch.nn.Parameter(torch.zeros(1, 100, hidden_dim))
22+
self.embedding = torch.nn.Embedding(vocab_size, hidden_dim)
23+
self.transformer = torch.nn.Transformer(
24+
d_model=hidden_dim,
25+
nhead=8,
26+
num_encoder_layers=3,
27+
num_decoder_layers=3,
28+
dim_feedforward=2048,
29+
batch_first=True,
30+
)
31+
self.fc_out = torch.nn.Linear(hidden_dim, vocab_size)
32+
33+
def forward(self, src_seq, tgt_in_seq):
34+
max_id = self.embedding.num_embeddings - 1
35+
src_seq = torch.clamp(src_seq, 0, max_id)
36+
tgt_in_seq = torch.clamp(tgt_in_seq, 0, max_id)
37+
max_pos = self.pos_encoder.size(1)
38+
src_seq = src_seq[:, :max_pos]
39+
tgt_in_seq = tgt_in_seq[:, :max_pos]
40+
41+
src_emb = self.embedding(src_seq) + self.pos_encoder[:, : src_seq.size(1), :]
42+
tgt_emb = self.embedding(tgt_in_seq) + self.pos_encoder[:, : tgt_in_seq.size(1), :]
43+
device = src_seq.device
44+
tgt_mask = torch.triu(
45+
torch.ones(tgt_in_seq.size(1), tgt_in_seq.size(1), dtype=torch.bool, device=device),
46+
diagonal=1,
47+
)
48+
out = self.transformer(src_emb, tgt_emb, tgt_mask=tgt_mask)
49+
logits = self.fc_out(out)
50+
return logits
51+
52+
1853
class CalculusSolverInference:
1954
def __init__(
2055
self,
@@ -39,38 +74,48 @@ def __init__(
3974
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
4075
rule_labels = self._load_rule_labels(vocab_path)
4176

42-
if model_path.endswith((".pt", ".pth")):
43-
from model.transformer import CalculusSolverModel
44-
hidden_dim = 128
45-
try:
46-
# Try to load hidden_dim from config.json in root
47-
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
48-
config_path = os.path.join(root_dir, "config.json")
49-
if os.path.exists(config_path):
50-
with open(config_path, "r") as f:
51-
cfg = json.load(f)
52-
hidden_dim = cfg.get("hidden_dim", 128)
53-
except Exception:
54-
pass
55-
vocab_size = max(self.vocab_map["token_to_id"].values()) + 1
56-
self.model = CalculusSolverModel(
57-
vocab_size=vocab_size,
58-
num_rules=len(rule_labels),
59-
hidden_dim=hidden_dim,
60-
rule_labels=rule_labels,
61-
).to(self.device)
77+
if isinstance(self.model_data, torch.nn.Module):
78+
self.model = self.model_data.to(self.device)
6279
else:
63-
self.model = CalculusModel(
64-
vocab_size=config.get("vocab_size", len(self.vocab_map["token_to_id"])),
65-
rule_labels=rule_labels,
66-
hidden_dim=config.get("hidden_dim", 512),
67-
num_heads=config.get("num_heads", 8),
68-
num_layers=config.get("num_layers", 8),
69-
ffn_dim=config.get("ffn_dim", 2048),
70-
dropout=config.get("dropout", 0.1),
71-
position_dim=config.get("position_dim", 3),
72-
).to(self.device)
73-
self.model.load_state_dict(self._resolve_state_dict(self.model_data))
80+
state_dict = self._resolve_state_dict(self.model_data)
81+
82+
if "fc_out.weight" in state_dict and "pos_encoder" in state_dict:
83+
vocab_size = state_dict["embedding.weight"].shape[0]
84+
hidden_dim = state_dict["embedding.weight"].shape[1]
85+
self.model = PklTransformerModel(vocab_size=vocab_size, hidden_dim=hidden_dim).to(self.device)
86+
elif any(k.startswith(("encoder.", "decoder.", "rule_head.")) for k in state_dict.keys()) and not model_path.endswith((".pt", ".pth")):
87+
self.model = CalculusModel(
88+
vocab_size=config.get("vocab_size", len(self.vocab_map["token_to_id"])),
89+
rule_labels=rule_labels,
90+
hidden_dim=config.get("hidden_dim", 512),
91+
num_heads=config.get("num_heads", 8),
92+
num_layers=config.get("num_layers", 8),
93+
ffn_dim=config.get("ffn_dim", 2048),
94+
dropout=config.get("dropout", 0.1),
95+
position_dim=config.get("position_dim", 3),
96+
).to(self.device)
97+
else:
98+
from model.transformer import CalculusSolverModel
99+
hidden_dim = 128
100+
try:
101+
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
102+
config_path = os.path.join(root_dir, "config.json")
103+
if os.path.exists(config_path):
104+
with open(config_path, "r") as f:
105+
cfg = json.load(f)
106+
hidden_dim = cfg.get("hidden_dim", 128)
107+
except Exception:
108+
pass
109+
vocab_size = max(self.vocab_map["token_to_id"].values()) + 1
110+
self.model = CalculusSolverModel(
111+
vocab_size=vocab_size,
112+
num_rules=len(rule_labels),
113+
hidden_dim=hidden_dim,
114+
rule_labels=rule_labels,
115+
).to(self.device)
116+
117+
self.model.load_state_dict(state_dict)
118+
74119
self.model.eval()
75120

76121
self.beam_size = beam_size
@@ -139,14 +184,6 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]:
139184
padded_tokens = token_ids + [self.pad_id] * (self.max_len - len(token_ids))
140185
src_tokens = torch.tensor([padded_tokens], dtype=torch.long, device=self.device)
141186

142-
# FIX: this environment's inference/beam_search.py has the
143-
# simplified signature (model, src_tokens, vocab_map, beam_size,
144-
# max_len, node_pool) -- it does not accept src_positions or
145-
# parent_child_pairs as external arguments. model/transformer.py's
146-
# CalculusSolverModel.forward() builds these zero-tensors
147-
# internally, so they were never needed here; passing them caused
148-
# "beam_search() got an unexpected keyword argument 'src_positions'"
149-
# on every single solve() call.
150187
result = beam_search(
151188
model=self.model,
152189
src_tokens=src_tokens,
@@ -162,26 +199,9 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]:
162199
if token_id in self.vocab_map["id_to_token"]
163200
]
164201

165-
# FIX (docs/KNOWN_ISSUES.md): beam_search seeds every beam with a
166-
# leading [BOS] token, which is correct for decoder input framing but
167-
# is not part of the SLaNg AST grammar itself. Downstream consumers
168-
# (the deserializer inside verify(), and any AST-structure parsing)
169-
# expect a pure token sequence starting at a real node type
170-
# (NODE:TERM / NODE:FRAC / OP:...), not [BOS]. Without this strip,
171-
# deserialization fails immediately with "Unexpected token ... [BOS]"
172-
# on every single call, regardless of whether the underlying sequence
173-
# the model generated was otherwise valid.
174202
if output_token_strings and output_token_strings[0] == "[BOS]":
175203
output_token_strings = output_token_strings[1:]
176204

177-
# FIX: this model folds rule prediction into the output sequence as
178-
# a leading RULE:xxx token (see docs/KNOWN_ISSUES.md, "Rule
179-
# prediction folded into output sequence"). It is not part of the
180-
# SLaNg AST grammar the verifier deserializes. Without this strip,
181-
# deserialization failed with "Unexpected token while parsing node
182-
# at index 0: RULE:partial_derivative" (or any other rule label)
183-
# on every single call, regardless of whether the rest of the
184-
# generated sequence was correct.
185205
predicted_rule = None
186206
if output_token_strings and output_token_strings[0].startswith("RULE:"):
187207
predicted_rule = output_token_strings[0]

0 commit comments

Comments
 (0)