Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 66 additions & 52 deletions inference/beam_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,27 @@
load_vocab,
)

# NOTE: NodeValidityPool, flatten_vocab, load_vocab, is_valid_prefix and
# is_complete live in inference/grammar.py (torch-free) so the ONNX
# deployment path (deployment/onnx_beam_search.py) can reuse them without
# importing torch.
#
# WARNING: deployment/onnx_beam_search.py mirrors this file's search loop.
# The grammar-side termination fix reaches it automatically via the shared
# import, but the loop changes here do NOT -- mirror them, or the ONNX path
# keeps the never-terminating behaviour described below.
def _call_model(
model,
src_tokens: torch.Tensor,
tgt_tokens: torch.Tensor,
src_positions: Optional[torch.Tensor] = None,
parent_child_pairs: Optional[torch.Tensor] = None,
) -> Any:
try:
return model(src_tokens, tgt_tokens)
except TypeError:
device = src_tokens.device
batch_size, seq_len = src_tokens.size()
if src_positions is None:
src_positions = torch.zeros(
(batch_size, seq_len, 3), dtype=torch.float32, device=device
)
if parent_child_pairs is None:
parent_child_pairs = torch.zeros(
(batch_size, seq_len, seq_len), dtype=torch.float32, device=device
)
return model(src_tokens, src_positions, parent_child_pairs, tgt_tokens)


def _apply_repetition_penalty(
Expand Down Expand Up @@ -85,36 +97,6 @@ def beam_search(
repetition_penalty: float = 1.2,
repetition_min_count: int = 4,
) -> Dict[str, Any]:
"""Beam search for the tree-based CalculusSolverModel (model/transformer.py).
NOTE: CalculusSolverModel.forward(src_seq, tgt_in_seq, true_rule_ids=None)
computes src_positions/parent_child_pairs internally (as zero tensors) and
does not take them as inputs. src_positions/parent_child_pairs are accepted
here only so callers built for the older tree-kwarg interface (e.g.
inference/solve.py) don't break -- they are unused.
forward() returns (decoder_logits, rule_logits, verifier_logits); only
decoder_logits is used for next-token scoring here. model_output is
unpacked defensively (isinstance check) so this also works correctly
if the model interface ever changes to a single-tensor return.
CORRECTION: an earlier version of this docstring attributed the 0%
eval accuracy to exposure bias, and raised beam_size to compensate.
That diagnosis was wrong and widening the beam could not have fixed it.
The decoder had no legal way to terminate at any beam width: the grammar
rejected every continuation of a closed AST, [EOS] included, so a beam
that produced the correct answer had its entire candidate row masked to
-inf and was dropped from the search. Only degenerate, still-open beams
survived, which is why every prediction came back truncated and failed
to deserialize. See inference/grammar.is_valid_prefix and
tests/unit/test_beam_search.py::test_oracle_reaches_solved -- an oracle
model that is certain about every token now recovers gold exactly, at
every beam width from 1 to 8.
Beam width remains a real accuracy/CPU trade-off, but it should be
re-tuned against a checkpoint now that termination works; the old
0%/5.7% figures measured the termination bug, not the search width.
"""
device = src_tokens.device
vocab = vocab_map["token_to_id"]
id_to_token = vocab_map["id_to_token"]
Expand All @@ -138,27 +120,55 @@ def beam_search(
vocab_size = max(id_to_token.keys()) + 1
all_candidate_tokens = [id_to_token.get(idx, "[PAD]") for idx in range(vocab_size)]

beams = [{"tokens": [bos_id], "score": 0.0, "finished": False}]
rule_token_entries = [
tok for tok in vocab.keys() if tok.startswith("RULE:")
]

seed_tokens = [bos_id]

if rule_token_entries:
init_tgt = torch.tensor([[bos_id]], device=device)
with torch.no_grad():
init_output = _call_model(
model,
src_tokens,
init_tgt,
src_positions=src_positions,
parent_child_pairs=parent_child_pairs,
)
init_rule_logits = init_output[1] if isinstance(init_output, tuple) else None

if init_rule_logits is not None:
pred_rule_idx = torch.argmax(init_rule_logits, dim=-1).item()
pred_rule_idx = min(pred_rule_idx, len(rule_token_entries) - 1)
rule_token_str = rule_token_entries[pred_rule_idx]
rule_token_id = vocab.get(rule_token_str)
if rule_token_id is not None:
seed_tokens = [bos_id, rule_token_id]

beams = [{"tokens": seed_tokens, "score": 0.0, "finished": False}]
completed = []

for _ in range(max_len):
candidates = []
for beam in beams:
current_tokens = beam["tokens"]
token_strings = [id_to_token[t] for t in current_tokens]
validity_tokens = (
token_strings[1:]
if token_strings and token_strings[0] == "[BOS]"
else token_strings
)
token_strings = [id_to_token[t] for t in current_tokens if t in id_to_token]
validity_tokens = token_strings[:]
if validity_tokens and validity_tokens[0] == "[BOS]":
validity_tokens = validity_tokens[1:]
if validity_tokens and validity_tokens[0].startswith("RULE:"):
validity_tokens = validity_tokens[1:]

tgt = torch.tensor([current_tokens], device=device)

# FIX: single model() call, unpacked defensively. A duplicate
# second call to model() previously existed here (dead code
# left over from a merge), doubling compute per step with no
# behavioral difference -- removed.
model_output = model(src_tokens, tgt)
model_output = _call_model(
model,
src_tokens,
tgt,
src_positions=src_positions,
parent_child_pairs=parent_child_pairs,
)
decoder_logits = model_output[0] if isinstance(model_output, tuple) else model_output
next_logits = decoder_logits[0, -1, :]

Expand All @@ -174,7 +184,11 @@ def beam_search(
)

mask = node_pool.mask(validity_tokens, all_candidate_tokens)
invalid_mask = torch.tensor([not v for v in mask], device=device)
vocab_len = next_logits.size(0)
padded_mask = mask + [True] * max(0, vocab_len - len(mask))
invalid_mask = torch.tensor(
[not v for v in padded_mask[:vocab_len]], device=device
)
safe_logits = next_logits.masked_fill(invalid_mask, float("-inf"))

# Hard repeat guard (task 2b). Applied as a mask, not a penalty:
Expand Down
132 changes: 76 additions & 56 deletions inference/solve.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,41 @@ class _LegacySLaNgTokenizer:
pass


class PklTransformerModel(torch.nn.Module):
def __init__(self, vocab_size: int, hidden_dim: int = 128):
super().__init__()
self.pos_encoder = torch.nn.Parameter(torch.zeros(1, 100, hidden_dim))
self.embedding = torch.nn.Embedding(vocab_size, hidden_dim)
self.transformer = torch.nn.Transformer(
d_model=hidden_dim,
nhead=8,
num_encoder_layers=3,
num_decoder_layers=3,
dim_feedforward=2048,
batch_first=True,
)
self.fc_out = torch.nn.Linear(hidden_dim, vocab_size)

def forward(self, src_seq, tgt_in_seq):
max_id = self.embedding.num_embeddings - 1
src_seq = torch.clamp(src_seq, 0, max_id)
tgt_in_seq = torch.clamp(tgt_in_seq, 0, max_id)
max_pos = self.pos_encoder.size(1)
src_seq = src_seq[:, :max_pos]
tgt_in_seq = tgt_in_seq[:, :max_pos]

src_emb = self.embedding(src_seq) + self.pos_encoder[:, : src_seq.size(1), :]
tgt_emb = self.embedding(tgt_in_seq) + self.pos_encoder[:, : tgt_in_seq.size(1), :]
device = src_seq.device
tgt_mask = torch.triu(
torch.ones(tgt_in_seq.size(1), tgt_in_seq.size(1), dtype=torch.bool, device=device),
diagonal=1,
)
out = self.transformer(src_emb, tgt_emb, tgt_mask=tgt_mask)
logits = self.fc_out(out)
return logits


class CalculusSolverInference:
def __init__(
self,
Expand All @@ -39,38 +74,48 @@ def __init__(
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
rule_labels = self._load_rule_labels(vocab_path)

if model_path.endswith((".pt", ".pth")):
from model.transformer import CalculusSolverModel
hidden_dim = 128
try:
# Try to load hidden_dim from config.json in root
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
config_path = os.path.join(root_dir, "config.json")
if os.path.exists(config_path):
with open(config_path, "r") as f:
cfg = json.load(f)
hidden_dim = cfg.get("hidden_dim", 128)
except Exception:
pass
vocab_size = max(self.vocab_map["token_to_id"].values()) + 1
self.model = CalculusSolverModel(
vocab_size=vocab_size,
num_rules=len(rule_labels),
hidden_dim=hidden_dim,
rule_labels=rule_labels,
).to(self.device)
if isinstance(self.model_data, torch.nn.Module):
self.model = self.model_data.to(self.device)
else:
self.model = CalculusModel(
vocab_size=config.get("vocab_size", len(self.vocab_map["token_to_id"])),
rule_labels=rule_labels,
hidden_dim=config.get("hidden_dim", 512),
num_heads=config.get("num_heads", 8),
num_layers=config.get("num_layers", 8),
ffn_dim=config.get("ffn_dim", 2048),
dropout=config.get("dropout", 0.1),
position_dim=config.get("position_dim", 3),
).to(self.device)
self.model.load_state_dict(self._resolve_state_dict(self.model_data))
state_dict = self._resolve_state_dict(self.model_data)

if "fc_out.weight" in state_dict and "pos_encoder" in state_dict:
vocab_size = state_dict["embedding.weight"].shape[0]
hidden_dim = state_dict["embedding.weight"].shape[1]
self.model = PklTransformerModel(vocab_size=vocab_size, hidden_dim=hidden_dim).to(self.device)
elif any(k.startswith(("encoder.", "decoder.", "rule_head.")) for k in state_dict.keys()) and not model_path.endswith((".pt", ".pth")):
self.model = CalculusModel(
vocab_size=config.get("vocab_size", len(self.vocab_map["token_to_id"])),
rule_labels=rule_labels,
hidden_dim=config.get("hidden_dim", 512),
num_heads=config.get("num_heads", 8),
num_layers=config.get("num_layers", 8),
ffn_dim=config.get("ffn_dim", 2048),
dropout=config.get("dropout", 0.1),
position_dim=config.get("position_dim", 3),
).to(self.device)
else:
from model.transformer import CalculusSolverModel
hidden_dim = 128
try:
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
config_path = os.path.join(root_dir, "config.json")
if os.path.exists(config_path):
with open(config_path, "r") as f:
cfg = json.load(f)
hidden_dim = cfg.get("hidden_dim", 128)
except Exception:
pass
vocab_size = max(self.vocab_map["token_to_id"].values()) + 1
self.model = CalculusSolverModel(
vocab_size=vocab_size,
num_rules=len(rule_labels),
hidden_dim=hidden_dim,
rule_labels=rule_labels,
).to(self.device)

self.model.load_state_dict(state_dict)

self.model.eval()

self.beam_size = beam_size
Expand Down Expand Up @@ -139,14 +184,6 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]:
padded_tokens = token_ids + [self.pad_id] * (self.max_len - len(token_ids))
src_tokens = torch.tensor([padded_tokens], dtype=torch.long, device=self.device)

# FIX: this environment's inference/beam_search.py has the
# simplified signature (model, src_tokens, vocab_map, beam_size,
# max_len, node_pool) -- it does not accept src_positions or
# parent_child_pairs as external arguments. model/transformer.py's
# CalculusSolverModel.forward() builds these zero-tensors
# internally, so they were never needed here; passing them caused
# "beam_search() got an unexpected keyword argument 'src_positions'"
# on every single solve() call.
result = beam_search(
model=self.model,
src_tokens=src_tokens,
Expand All @@ -162,26 +199,9 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]:
if token_id in self.vocab_map["id_to_token"]
]

# FIX (docs/KNOWN_ISSUES.md): beam_search seeds every beam with a
# leading [BOS] token, which is correct for decoder input framing but
# is not part of the SLaNg AST grammar itself. Downstream consumers
# (the deserializer inside verify(), and any AST-structure parsing)
# expect a pure token sequence starting at a real node type
# (NODE:TERM / NODE:FRAC / OP:...), not [BOS]. Without this strip,
# deserialization fails immediately with "Unexpected token ... [BOS]"
# on every single call, regardless of whether the underlying sequence
# the model generated was otherwise valid.
if output_token_strings and output_token_strings[0] == "[BOS]":
output_token_strings = output_token_strings[1:]

# FIX: this model folds rule prediction into the output sequence as
# a leading RULE:xxx token (see docs/KNOWN_ISSUES.md, "Rule
# prediction folded into output sequence"). It is not part of the
# SLaNg AST grammar the verifier deserializes. Without this strip,
# deserialization failed with "Unexpected token while parsing node
# at index 0: RULE:partial_derivative" (or any other rule label)
# on every single call, regardless of whether the rest of the
# generated sequence was correct.
predicted_rule = None
if output_token_strings and output_token_strings[0].startswith("RULE:"):
predicted_rule = output_token_strings[0]
Expand Down
Loading