Skip to content

Commit cf95350

Browse files
authored
Merge pull request #44 from mustfaaaa/dev2/beam-search-repetition-control
fix(beam-search): make decode terminable, bound the grammar, add repe…
2 parents 9f661b6 + f466f76 commit cf95350

15 files changed

Lines changed: 2334 additions & 78 deletions

‎deployment/onnx_beam_search.py‎

Lines changed: 105 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,51 @@
11
"""
22
numpy/onnxruntime-only mirror of inference/beam_search.py -- mirrors that
3-
file's beam_search() line-for-line in logic, but never imports torch.
4-
This is the entire point of the Option A (ONNX) deployment path: the
5-
production Vercel bundle only needs onnxruntime + numpy, not the full
6-
PyTorch package, which is what pushed the old bundle over the ~250MB
7-
serverless size limit.
3+
file in logic, but never imports torch. This is the entire point of the
4+
Option A (ONNX) deployment path: the production Vercel bundle only needs
5+
onnxruntime + numpy, not the full PyTorch package, which is what pushed the
6+
old bundle over the ~250MB serverless size limit.
7+
8+
The two files had drifted badly -- every termination and repetition fix
9+
landed on the torch side only. The shared policy now lives in
10+
inference/decoding.py (pure stdlib) so they cannot silently diverge again;
11+
only the array arithmetic differs between them.
812
"""
913

1014
from typing import Any, Dict, List, Optional
1115

1216
import numpy as np
1317
import onnxruntime as ort
1418

15-
from inference.grammar import NodeValidityPool
16-
17-
18-
def _softmax(x: np.ndarray) -> np.ndarray:
19-
x = x - np.max(x)
20-
e = np.exp(x)
21-
return e / e.sum()
19+
from inference.decoding import (
20+
content_token_ids,
21+
escape_token_ids,
22+
longest_complete_prefix,
23+
penalised_logit,
24+
repeat_blocked_tokens,
25+
repetition_penalty_targets,
26+
strip_terminator,
27+
)
28+
from inference.grammar import NodeValidityPool, is_complete
29+
30+
31+
def _log_softmax(x: np.ndarray) -> np.ndarray:
32+
"""Numerically stable log-softmax that PRESERVES -inf for masked entries.
33+
34+
The previous implementation was np.log(softmax(x) + 1e-12), which mapped
35+
every grammatically illegal token to log(1e-12) ~= -27.6 -- a FINITE
36+
score. argpartition would then happily select masked tokens, and since
37+
[EOS] is one of them, that manufactured "finished" beams holding a
38+
truncated AST. A numpy-only hazard: the torch path gets true -inf from
39+
torch.log_softmax and never had it.
40+
"""
41+
finite = np.isfinite(x)
42+
out = np.full(x.shape, -np.inf, dtype=np.float64)
43+
if not finite.any():
44+
return out
45+
shifted = x - np.max(x[finite])
46+
exponentiated = np.where(finite, np.exp(shifted), 0.0)
47+
out[finite] = shifted[finite] - np.log(exponentiated.sum())
48+
return out
2249

2350

2451
def onnx_beam_search(
@@ -28,8 +55,12 @@ def onnx_beam_search(
2855
beam_size: int = 5,
2956
max_len: int = 32,
3057
node_pool: Optional[NodeValidityPool] = None,
58+
max_token_run: int = 4,
59+
no_repeat_ngram_size: int = 2,
60+
repetition_penalty: float = 1.2,
61+
repetition_min_count: int = 4,
3162
) -> Dict[str, Any]:
32-
"""Mirrors inference/beam_search.py::beam_search(), but calls the
63+
"""Mirrors inference/beam_search.py::beam_search(), but drives the
3364
exported ONNX graph via onnxruntime instead of a torch.nn.Module."""
3465
vocab = vocab_map["token_to_id"]
3566
id_to_token = vocab_map["id_to_token"]
@@ -39,21 +70,20 @@ def onnx_beam_search(
3970
if node_pool is None:
4071
node_pool = NodeValidityPool()
4172

73+
exempt_ids = escape_token_ids(vocab)
74+
content_ids = content_token_ids(id_to_token)
75+
4276
vocab_size = max(id_to_token.keys()) + 1
4377
all_candidate_tokens = [id_to_token.get(idx, "[PAD]") for idx in range(vocab_size)]
4478

4579
src_arr = np.array([src_tokens], dtype=np.int64)
4680

4781
beams = [{"tokens": [bos_id], "score": 0.0, "finished": False}]
48-
completed = []
82+
completed: List[Dict[str, Any]] = []
4983

5084
for _ in range(max_len):
51-
candidates = []
85+
candidates: List[Dict[str, Any]] = []
5286
for beam in beams:
53-
if beam["finished"]:
54-
candidates.append(beam)
55-
continue
56-
5787
current_tokens = beam["tokens"]
5888
token_strings = [id_to_token[t] for t in current_tokens]
5989
validity_tokens = (
@@ -67,42 +97,83 @@ def onnx_beam_search(
6797
["logits"],
6898
{"src_seq": src_arr, "tgt_in_seq": tgt_arr},
6999
)[0]
70-
next_logits = logits[0, -1, :]
100+
next_logits = np.asarray(logits[0, -1, :], dtype=np.float64).copy()
101+
102+
if repetition_penalty != 1.0:
103+
for token_id in repetition_penalty_targets(
104+
current_tokens, repetition_min_count, exempt_ids, content_ids
105+
):
106+
if 0 <= token_id < next_logits.shape[0]:
107+
next_logits[token_id] = penalised_logit(
108+
float(next_logits[token_id]), repetition_penalty
109+
)
71110

72111
mask = node_pool.mask(validity_tokens, all_candidate_tokens)
73112
safe_logits = next_logits.copy()
74113
safe_logits[[not v for v in mask]] = -np.inf
75114

76-
if np.all(np.isinf(safe_logits)):
115+
blocked = repeat_blocked_tokens(
116+
current_tokens,
117+
max_token_run,
118+
no_repeat_ngram_size,
119+
exempt_ids,
120+
content_ids,
121+
)
122+
if blocked:
123+
guarded = safe_logits.copy()
124+
for blocked_id in blocked:
125+
if 0 <= blocked_id < guarded.shape[0]:
126+
guarded[blocked_id] = -np.inf
127+
if np.isfinite(guarded).any():
128+
safe_logits = guarded
129+
130+
valid_count = int(np.isfinite(safe_logits).sum())
131+
if valid_count == 0:
132+
if is_complete(validity_tokens):
133+
completed.append({
134+
"tokens": current_tokens + [eos_id],
135+
"score": beam["score"],
136+
"finished": True,
137+
})
77138
continue
78139

79-
log_probs = np.log(_softmax(safe_logits) + 1e-12)
80-
k = min(beam_size, safe_logits.shape[0])
140+
log_probs = _log_softmax(safe_logits)
141+
142+
k = min(beam_size, valid_count, log_probs.shape[0])
81143
top_idx = np.argpartition(-log_probs, k - 1)[:k]
82144
top_idx = top_idx[np.argsort(-log_probs[top_idx])]
83145

84-
for token_id in top_idx:
85-
token_id = int(token_id)
146+
for raw_id in top_idx:
147+
token_id = int(raw_id)
86148
score = float(log_probs[token_id])
87-
new_tokens = current_tokens + [token_id]
88-
finished = token_id == eos_id
89-
candidates.append({
90-
"tokens": new_tokens,
149+
if not np.isfinite(score):
150+
continue
151+
new_beam = {
152+
"tokens": current_tokens + [token_id],
91153
"score": beam["score"] + score,
92-
"finished": finished,
93-
})
154+
"finished": token_id == eos_id,
155+
}
156+
target = completed if new_beam["finished"] else candidates
157+
target.append(new_beam)
94158

95159
if not candidates:
96160
break
97161

98162
beams = sorted(candidates, key=lambda x: x["score"], reverse=True)[:beam_size]
99-
if all(b["finished"] for b in beams):
100-
completed.extend(beams)
163+
164+
if completed and beams[0]["score"] <= max(c["score"] for c in completed):
101165
break
102166

103167
best = sorted(completed, key=lambda x: x["score"], reverse=True)[0] if completed else (
104168
beams[0] if beams else {"tokens": [bos_id], "score": 0.0, "finished": False}
105169
)
106170

107171
status = "solved" if best["finished"] else "partial"
108-
return {"tokens": best["tokens"], "score": best["score"], "status": status}
172+
tokens = strip_terminator(best["tokens"], eos_id)
173+
174+
if not best["finished"]:
175+
salvaged = longest_complete_prefix(tokens, id_to_token)
176+
if salvaged:
177+
tokens = salvaged
178+
179+
return {"tokens": tokens, "score": best["score"], "status": status}

‎docs/BEAM_SEARCH_PERFORMANCE.md‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# Beam Search Performance — validity-mask cost and the eval runtime bound
2+
3+
Answers DEV 2 task 3. Reproduce with:
4+
5+
```bash
6+
python scripts/profile_beam.py --problems 8 --beams 1,2,5 --max-len 256
7+
```
8+
9+
## Method
10+
11+
Forward-pass **count** is exact. Forward-pass **wall time** is projected from a
12+
separately measured per-call cost on the real architecture
13+
(`CalculusSolverModel`, `hidden_dim=256`, 4 threads, `src_len=48`,
14+
**~24 ms/call**); timing every call in-loop would add tens of minutes and
15+
measure nothing extra, since cost per call does not depend on weight values.
16+
Mask time is measured in-loop and is real.
17+
18+
There is no checkpoint in this repo, and decode **length** is model-dependent,
19+
so two behaviours bracket reality:
20+
21+
| behaviour | meaning |
22+
|---|---|
23+
| **oracle** | certain about every gold token; terminates correctly. What a trained model should approach. |
24+
| **untrained** | uniform random logits; wanders. The pessimistic bound. |
25+
26+
## 1. Validity-mask cost at beam_size=5
27+
28+
All figures at `max_len=256`, which is what `run_eval.py` actually uses
29+
(see §3).
30+
31+
| behaviour | beam | guards | is_valid_prefix calls | tokens parsed | mask ms | forwards | proj. total s | **mask share** |
32+
|---|---|---|---|---|---|---|---|---|
33+
| oracle | 1 | on | 2,806 | 36,410 | 27 | 23 | 0.60 | 4.4% |
34+
| oracle | 2 | on | 5,487 | 72,695 | 44 | 44 | 1.17 | 3.8% |
35+
| oracle | 5 | on | 13,532 | 181,552 | 120 | 109 | 2.89 | 4.2% |
36+
| untrained | 1 | on | 6,076 | 151,900 | 78 | 49 | 1.32 | 5.9% |
37+
| untrained | 2 | on | 15,252 | 484,220 | 205 | 123 | 3.32 | 6.2% |
38+
| untrained | 5 | on | 27,404 | 641,204 | 337 | 221 | 5.94 | 5.7% |
39+
40+
**The mask is not the bottleneck.** It is 4–6% of runtime; the un-batched
41+
forward passes are the rest. Beam width scales both terms roughly linearly, so
42+
`beam_size=2 -> 5` costs about 2.5x across the board — the mask is not what
43+
makes width expensive.
44+
45+
## 2. Effect of the repetition guards on mask cost
46+
47+
The guards never increase `is_valid_prefix` volume. They are either neutral or
48+
a small saving:
49+
50+
| behaviour | beam | guards on | guards off |
51+
|---|---|---|---|
52+
| oracle | 5 | 13,532 | 13,532 (identical) |
53+
| untrained | 1 | 6,076 | 6,448 |
54+
| untrained | 5 | 27,404 | 29,884 |
55+
56+
Their own cost is **0.0084 ms/step** — 0.02% of a step, against ~24 ms for a
57+
forward pass.
58+
59+
> **Correction.** An earlier revision of this document claimed call volume was
60+
> *identical* with guards on and off, and described that as structural. That was
61+
> over-generalised from two configurations. Volume is driven by
62+
> `beams x steps x |V|`, and the guards do change how many steps a degenerate
63+
> beam survives — so they can reduce it. The claim to rely on is the weaker,
64+
> true one: **the guards never make mask cost worse.**
65+
66+
## 3. The `eval/run_eval.py` bound
67+
68+
`run_eval.py` constructs `CalculusSolverInference(model_path=..., beam_size=5)`
69+
with no `max_len`, so it inherits **256** from `inference/solve.py`. Over the
70+
300 benchmark problems:
71+
72+
| model behaviour | per problem | **300 problems** |
73+
|---|---|---|
74+
| correct (terminates) | 2.89 s | **~15 minutes** |
75+
| worst case (never terminates) | 5.94 s | **~30 minutes** |
76+
77+
**Documented bound: `run_eval.py` at `beam_size=5` should complete in ~15
78+
minutes, and must not exceed ~30 minutes.** Materially longer than that means
79+
something regressed in the grammar bounds of §4 — it is not the mask.
80+
81+
This bound is projected, not measured end-to-end: `run_eval.py` exits
82+
immediately because `checkpoints/final/best.pt` is absent. Re-run
83+
`scripts/profile_beam.py` once a checkpoint exists to close the gap.
84+
85+
## 4. Structural bounds are what made the worst case tractable
86+
87+
Before the grammar was bounded, the untrained worst case at `beam_size=5`,
88+
`max_len=256` was **34.58 s/problem** — 158,224 `is_valid_prefix` calls and
89+
**20.4 million tokens parsed**, with the mask at 14.7%. `is_valid_prefix`
90+
re-parses the whole prefix per candidate, so cost is
91+
`O(beams x steps x |V| x L)` — quadratic in sequence length. A decoder that
92+
never terminates drags every beam to `L=256` and pays that quadratic.
93+
94+
Capping the grammar's previously unbounded dimensions removed it:
95+
96+
| bound | max in 155,000 real targets | cap | margin |
97+
|---|---|---|---|
98+
| `MAX_OPVARS_PER_OP` | 1 | 4 | 4x |
99+
| `MAX_NESTING_DEPTH` | 3 | 8 | 2.7x |
100+
| `MAX_SIBLINGS` | 2 | 8 | 4x |
101+
102+
Result: **34.58 s -> 5.94 s per problem** (5.8x), tokens parsed 20.4M -> 641k
103+
(32x). Grammar acceptance of real targets is unchanged at **100.00%** of all
104+
155,000 — the caps reject nothing real.
105+
106+
## 5. Known limitation
107+
108+
The bounds do **not** guarantee that a degenerate decoder terminates within
109+
`max_len`. A tree of depth 8 with 8 siblings per node can hold far more than
110+
256 tokens, so a pathological model can still fill the budget with a legal but
111+
incomplete prefix and be truncated. Measured: a model biased toward nested
112+
`NODE:FRAC` still reaches `max_len` with `status="partial"`.
113+
114+
`beam_search` mitigates but cannot eliminate this: when no beam terminates it
115+
returns the longest **closed** prefix (`longest_complete_prefix`), so the caller
116+
gets something parseable. That only helps when the outermost node closed — a
117+
sequence that opens a fraction and never closes it has no complete prefix, and
118+
is returned as-is.
119+
120+
Guaranteeing termination would require a budget-aware constraint: once the
121+
remaining budget equals the tokens needed to close all open structures, mask
122+
everything except closing tokens. That is implementable but changes what the
123+
model is allowed to emit, so it is flagged rather than assumed.
124+
125+
## 6. Recommendation
126+
127+
**Decouple the decoder step budget from the encoder padding width.**
128+
`inference/solve.py` uses one `max_len=256` for both source padding
129+
(`solve.py:137-139`) and the generation budget (`solve.py:155`). They are
130+
unrelated quantities.
131+
132+
The longest serialized target across all 155,000 dataset targets is **38
133+
tokens**. A decoder budget of **48** (already the value in `config.json`) covers
134+
100% of real targets with 9 tokens spare. This caps the worst case directly
135+
rather than relying on the model to stop.
136+
137+
Secondary: batching the beams into one forward call per step was measured at
138+
only **~1.5x** (373ms -> 254ms for 5 beams), not the 5x it looks like — the
139+
model is CPU-bound and does not parallelise well across batch at 4 threads.
140+
Worth doing, but it is not the lever.

0 commit comments

Comments
 (0)