11"""
22numpy/onnxruntime-only mirror of inference/beam_search.py -- mirrors that
3- file's beam_search() line-for-line in logic, but never imports torch.
4- This is the entire point of the Option A (ONNX) deployment path: the
5- production Vercel bundle only needs onnxruntime + numpy, not the full
6- PyTorch package, which is what pushed the old bundle over the ~250MB
7- serverless size limit.
3+ file in logic, but never imports torch. This is the entire point of the
4+ Option A (ONNX) deployment path: the production Vercel bundle only needs
5+ onnxruntime + numpy, not the full PyTorch package, which is what pushed the
6+ old bundle over the ~250MB serverless size limit.
7+
8+ The two files had drifted badly -- every termination and repetition fix
9+ landed on the torch side only. The shared policy now lives in
10+ inference/decoding.py (pure stdlib) so they cannot silently diverge again;
11+ only the array arithmetic differs between them.
812"""
913
1014from typing import Any , Dict , List , Optional
1115
1216import numpy as np
1317import onnxruntime as ort
1418
15- from inference .grammar import NodeValidityPool
16-
17-
18- def _softmax (x : np .ndarray ) -> np .ndarray :
19- x = x - np .max (x )
20- e = np .exp (x )
21- return e / e .sum ()
19+ from inference .decoding import (
20+ content_token_ids ,
21+ escape_token_ids ,
22+ longest_complete_prefix ,
23+ penalised_logit ,
24+ repeat_blocked_tokens ,
25+ repetition_penalty_targets ,
26+ strip_terminator ,
27+ )
28+ from inference .grammar import NodeValidityPool , is_complete
29+
30+
31+ def _log_softmax (x : np .ndarray ) -> np .ndarray :
32+ """Numerically stable log-softmax that PRESERVES -inf for masked entries.
33+
34+ The previous implementation was np.log(softmax(x) + 1e-12), which mapped
35+ every grammatically illegal token to log(1e-12) ~= -27.6 -- a FINITE
36+ score. argpartition would then happily select masked tokens, and since
37+ [EOS] is one of them, that manufactured "finished" beams holding a
38+ truncated AST. A numpy-only hazard: the torch path gets true -inf from
39+ torch.log_softmax and never had it.
40+ """
41+ finite = np .isfinite (x )
42+ out = np .full (x .shape , - np .inf , dtype = np .float64 )
43+ if not finite .any ():
44+ return out
45+ shifted = x - np .max (x [finite ])
46+ exponentiated = np .where (finite , np .exp (shifted ), 0.0 )
47+ out [finite ] = shifted [finite ] - np .log (exponentiated .sum ())
48+ return out
2249
2350
2451def onnx_beam_search (
@@ -28,8 +55,12 @@ def onnx_beam_search(
2855 beam_size : int = 5 ,
2956 max_len : int = 32 ,
3057 node_pool : Optional [NodeValidityPool ] = None ,
58+ max_token_run : int = 4 ,
59+ no_repeat_ngram_size : int = 2 ,
60+ repetition_penalty : float = 1.2 ,
61+ repetition_min_count : int = 4 ,
3162) -> Dict [str , Any ]:
32- """Mirrors inference/beam_search.py::beam_search(), but calls the
63+ """Mirrors inference/beam_search.py::beam_search(), but drives the
3364 exported ONNX graph via onnxruntime instead of a torch.nn.Module."""
3465 vocab = vocab_map ["token_to_id" ]
3566 id_to_token = vocab_map ["id_to_token" ]
@@ -39,21 +70,20 @@ def onnx_beam_search(
3970 if node_pool is None :
4071 node_pool = NodeValidityPool ()
4172
73+ exempt_ids = escape_token_ids (vocab )
74+ content_ids = content_token_ids (id_to_token )
75+
4276 vocab_size = max (id_to_token .keys ()) + 1
4377 all_candidate_tokens = [id_to_token .get (idx , "[PAD]" ) for idx in range (vocab_size )]
4478
4579 src_arr = np .array ([src_tokens ], dtype = np .int64 )
4680
4781 beams = [{"tokens" : [bos_id ], "score" : 0.0 , "finished" : False }]
48- completed = []
82+ completed : List [ Dict [ str , Any ]] = []
4983
5084 for _ in range (max_len ):
51- candidates = []
85+ candidates : List [ Dict [ str , Any ]] = []
5286 for beam in beams :
53- if beam ["finished" ]:
54- candidates .append (beam )
55- continue
56-
5787 current_tokens = beam ["tokens" ]
5888 token_strings = [id_to_token [t ] for t in current_tokens ]
5989 validity_tokens = (
@@ -67,42 +97,83 @@ def onnx_beam_search(
6797 ["logits" ],
6898 {"src_seq" : src_arr , "tgt_in_seq" : tgt_arr },
6999 )[0 ]
70- next_logits = logits [0 , - 1 , :]
100+ next_logits = np .asarray (logits [0 , - 1 , :], dtype = np .float64 ).copy ()
101+
102+ if repetition_penalty != 1.0 :
103+ for token_id in repetition_penalty_targets (
104+ current_tokens , repetition_min_count , exempt_ids , content_ids
105+ ):
106+ if 0 <= token_id < next_logits .shape [0 ]:
107+ next_logits [token_id ] = penalised_logit (
108+ float (next_logits [token_id ]), repetition_penalty
109+ )
71110
72111 mask = node_pool .mask (validity_tokens , all_candidate_tokens )
73112 safe_logits = next_logits .copy ()
74113 safe_logits [[not v for v in mask ]] = - np .inf
75114
76- if np .all (np .isinf (safe_logits )):
115+ blocked = repeat_blocked_tokens (
116+ current_tokens ,
117+ max_token_run ,
118+ no_repeat_ngram_size ,
119+ exempt_ids ,
120+ content_ids ,
121+ )
122+ if blocked :
123+ guarded = safe_logits .copy ()
124+ for blocked_id in blocked :
125+ if 0 <= blocked_id < guarded .shape [0 ]:
126+ guarded [blocked_id ] = - np .inf
127+ if np .isfinite (guarded ).any ():
128+ safe_logits = guarded
129+
130+ valid_count = int (np .isfinite (safe_logits ).sum ())
131+ if valid_count == 0 :
132+ if is_complete (validity_tokens ):
133+ completed .append ({
134+ "tokens" : current_tokens + [eos_id ],
135+ "score" : beam ["score" ],
136+ "finished" : True ,
137+ })
77138 continue
78139
79- log_probs = np .log (_softmax (safe_logits ) + 1e-12 )
80- k = min (beam_size , safe_logits .shape [0 ])
140+ log_probs = _log_softmax (safe_logits )
141+
142+ k = min (beam_size , valid_count , log_probs .shape [0 ])
81143 top_idx = np .argpartition (- log_probs , k - 1 )[:k ]
82144 top_idx = top_idx [np .argsort (- log_probs [top_idx ])]
83145
84- for token_id in top_idx :
85- token_id = int (token_id )
146+ for raw_id in top_idx :
147+ token_id = int (raw_id )
86148 score = float (log_probs [token_id ])
87- new_tokens = current_tokens + [ token_id ]
88- finished = token_id == eos_id
89- candidates . append ( {
90- "tokens" : new_tokens ,
149+ if not np . isfinite ( score ):
150+ continue
151+ new_beam = {
152+ "tokens" : current_tokens + [ token_id ] ,
91153 "score" : beam ["score" ] + score ,
92- "finished" : finished ,
93- })
154+ "finished" : token_id == eos_id ,
155+ }
156+ target = completed if new_beam ["finished" ] else candidates
157+ target .append (new_beam )
94158
95159 if not candidates :
96160 break
97161
98162 beams = sorted (candidates , key = lambda x : x ["score" ], reverse = True )[:beam_size ]
99- if all ( b [ "finished" ] for b in beams ):
100- completed . extend ( beams )
163+
164+ if completed and beams [ 0 ][ "score" ] <= max ( c [ "score" ] for c in completed ):
101165 break
102166
103167 best = sorted (completed , key = lambda x : x ["score" ], reverse = True )[0 ] if completed else (
104168 beams [0 ] if beams else {"tokens" : [bos_id ], "score" : 0.0 , "finished" : False }
105169 )
106170
107171 status = "solved" if best ["finished" ] else "partial"
108- return {"tokens" : best ["tokens" ], "score" : best ["score" ], "status" : status }
172+ tokens = strip_terminator (best ["tokens" ], eos_id )
173+
174+ if not best ["finished" ]:
175+ salvaged = longest_complete_prefix (tokens , id_to_token )
176+ if salvaged :
177+ tokens = salvaged
178+
179+ return {"tokens" : tokens , "score" : best ["score" ], "status" : status }
0 commit comments