@@ -15,6 +15,41 @@ class _LegacySLaNgTokenizer:
1515 pass
1616
1717
18+ class PklTransformerModel (torch .nn .Module ):
19+ def __init__ (self , vocab_size : int , hidden_dim : int = 128 ):
20+ super ().__init__ ()
21+ self .pos_encoder = torch .nn .Parameter (torch .zeros (1 , 100 , hidden_dim ))
22+ self .embedding = torch .nn .Embedding (vocab_size , hidden_dim )
23+ self .transformer = torch .nn .Transformer (
24+ d_model = hidden_dim ,
25+ nhead = 8 ,
26+ num_encoder_layers = 3 ,
27+ num_decoder_layers = 3 ,
28+ dim_feedforward = 2048 ,
29+ batch_first = True ,
30+ )
31+ self .fc_out = torch .nn .Linear (hidden_dim , vocab_size )
32+
33+ def forward (self , src_seq , tgt_in_seq ):
34+ max_id = self .embedding .num_embeddings - 1
35+ src_seq = torch .clamp (src_seq , 0 , max_id )
36+ tgt_in_seq = torch .clamp (tgt_in_seq , 0 , max_id )
37+ max_pos = self .pos_encoder .size (1 )
38+ src_seq = src_seq [:, :max_pos ]
39+ tgt_in_seq = tgt_in_seq [:, :max_pos ]
40+
41+ src_emb = self .embedding (src_seq ) + self .pos_encoder [:, : src_seq .size (1 ), :]
42+ tgt_emb = self .embedding (tgt_in_seq ) + self .pos_encoder [:, : tgt_in_seq .size (1 ), :]
43+ device = src_seq .device
44+ tgt_mask = torch .triu (
45+ torch .ones (tgt_in_seq .size (1 ), tgt_in_seq .size (1 ), dtype = torch .bool , device = device ),
46+ diagonal = 1 ,
47+ )
48+ out = self .transformer (src_emb , tgt_emb , tgt_mask = tgt_mask )
49+ logits = self .fc_out (out )
50+ return logits
51+
52+
1853class CalculusSolverInference :
1954 def __init__ (
2055 self ,
@@ -39,38 +74,48 @@ def __init__(
3974 self .device = torch .device ("cuda" if torch .cuda .is_available () else "cpu" )
4075 rule_labels = self ._load_rule_labels (vocab_path )
4176
42- if model_path .endswith ((".pt" , ".pth" )):
43- from model .transformer import CalculusSolverModel
44- hidden_dim = 128
45- try :
46- # Try to load hidden_dim from config.json in root
47- root_dir = os .path .dirname (os .path .dirname (os .path .abspath (__file__ )))
48- config_path = os .path .join (root_dir , "config.json" )
49- if os .path .exists (config_path ):
50- with open (config_path , "r" ) as f :
51- cfg = json .load (f )
52- hidden_dim = cfg .get ("hidden_dim" , 128 )
53- except Exception :
54- pass
55- vocab_size = max (self .vocab_map ["token_to_id" ].values ()) + 1
56- self .model = CalculusSolverModel (
57- vocab_size = vocab_size ,
58- num_rules = len (rule_labels ),
59- hidden_dim = hidden_dim ,
60- rule_labels = rule_labels ,
61- ).to (self .device )
77+ if isinstance (self .model_data , torch .nn .Module ):
78+ self .model = self .model_data .to (self .device )
6279 else :
63- self .model = CalculusModel (
64- vocab_size = config .get ("vocab_size" , len (self .vocab_map ["token_to_id" ])),
65- rule_labels = rule_labels ,
66- hidden_dim = config .get ("hidden_dim" , 512 ),
67- num_heads = config .get ("num_heads" , 8 ),
68- num_layers = config .get ("num_layers" , 8 ),
69- ffn_dim = config .get ("ffn_dim" , 2048 ),
70- dropout = config .get ("dropout" , 0.1 ),
71- position_dim = config .get ("position_dim" , 3 ),
72- ).to (self .device )
73- self .model .load_state_dict (self ._resolve_state_dict (self .model_data ))
80+ state_dict = self ._resolve_state_dict (self .model_data )
81+
82+ if "fc_out.weight" in state_dict and "pos_encoder" in state_dict :
83+ vocab_size = state_dict ["embedding.weight" ].shape [0 ]
84+ hidden_dim = state_dict ["embedding.weight" ].shape [1 ]
85+ self .model = PklTransformerModel (vocab_size = vocab_size , hidden_dim = hidden_dim ).to (self .device )
86+ elif any (k .startswith (("encoder." , "decoder." , "rule_head." )) for k in state_dict .keys ()) and not model_path .endswith ((".pt" , ".pth" )):
87+ self .model = CalculusModel (
88+ vocab_size = config .get ("vocab_size" , len (self .vocab_map ["token_to_id" ])),
89+ rule_labels = rule_labels ,
90+ hidden_dim = config .get ("hidden_dim" , 512 ),
91+ num_heads = config .get ("num_heads" , 8 ),
92+ num_layers = config .get ("num_layers" , 8 ),
93+ ffn_dim = config .get ("ffn_dim" , 2048 ),
94+ dropout = config .get ("dropout" , 0.1 ),
95+ position_dim = config .get ("position_dim" , 3 ),
96+ ).to (self .device )
97+ else :
98+ from model .transformer import CalculusSolverModel
99+ hidden_dim = 128
100+ try :
101+ root_dir = os .path .dirname (os .path .dirname (os .path .abspath (__file__ )))
102+ config_path = os .path .join (root_dir , "config.json" )
103+ if os .path .exists (config_path ):
104+ with open (config_path , "r" ) as f :
105+ cfg = json .load (f )
106+ hidden_dim = cfg .get ("hidden_dim" , 128 )
107+ except Exception :
108+ pass
109+ vocab_size = max (self .vocab_map ["token_to_id" ].values ()) + 1
110+ self .model = CalculusSolverModel (
111+ vocab_size = vocab_size ,
112+ num_rules = len (rule_labels ),
113+ hidden_dim = hidden_dim ,
114+ rule_labels = rule_labels ,
115+ ).to (self .device )
116+
117+ self .model .load_state_dict (state_dict )
118+
74119 self .model .eval ()
75120
76121 self .beam_size = beam_size
@@ -139,14 +184,6 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]:
139184 padded_tokens = token_ids + [self .pad_id ] * (self .max_len - len (token_ids ))
140185 src_tokens = torch .tensor ([padded_tokens ], dtype = torch .long , device = self .device )
141186
142- # FIX: this environment's inference/beam_search.py has the
143- # simplified signature (model, src_tokens, vocab_map, beam_size,
144- # max_len, node_pool) -- it does not accept src_positions or
145- # parent_child_pairs as external arguments. model/transformer.py's
146- # CalculusSolverModel.forward() builds these zero-tensors
147- # internally, so they were never needed here; passing them caused
148- # "beam_search() got an unexpected keyword argument 'src_positions'"
149- # on every single solve() call.
150187 result = beam_search (
151188 model = self .model ,
152189 src_tokens = src_tokens ,
@@ -162,26 +199,9 @@ def solve(self, input_env: Dict[str, Any]) -> Dict[str, Any]:
162199 if token_id in self .vocab_map ["id_to_token" ]
163200 ]
164201
165- # FIX (docs/KNOWN_ISSUES.md): beam_search seeds every beam with a
166- # leading [BOS] token, which is correct for decoder input framing but
167- # is not part of the SLaNg AST grammar itself. Downstream consumers
168- # (the deserializer inside verify(), and any AST-structure parsing)
169- # expect a pure token sequence starting at a real node type
170- # (NODE:TERM / NODE:FRAC / OP:...), not [BOS]. Without this strip,
171- # deserialization fails immediately with "Unexpected token ... [BOS]"
172- # on every single call, regardless of whether the underlying sequence
173- # the model generated was otherwise valid.
174202 if output_token_strings and output_token_strings [0 ] == "[BOS]" :
175203 output_token_strings = output_token_strings [1 :]
176204
177- # FIX: this model folds rule prediction into the output sequence as
178- # a leading RULE:xxx token (see docs/KNOWN_ISSUES.md, "Rule
179- # prediction folded into output sequence"). It is not part of the
180- # SLaNg AST grammar the verifier deserializes. Without this strip,
181- # deserialization failed with "Unexpected token while parsing node
182- # at index 0: RULE:partial_derivative" (or any other rule label)
183- # on every single call, regardless of whether the rest of the
184- # generated sequence was correct.
185205 predicted_rule = None
186206 if output_token_strings and output_token_strings [0 ].startswith ("RULE:" ):
187207 predicted_rule = output_token_strings [0 ]
0 commit comments