Skip to content

Commit 5f61a92

Browse files
Hadar301claude
authored andcommitted
feat: add direct per-token latency test to measure guardrails overhead
Bypasses OpenCode's event batching and times raw SSE chunks from the MaaS gateway directly. Used to confirm inter-token intervals are identical with/without guardrails (~9ms); only TTFT differs. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
1 parent 6c8c2ce commit 5f61a92

1 file changed

Lines changed: 180 additions & 0 deletions

File tree

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
"""Direct gateway inter-token latency test.
2+
3+
Bypasses OpenCode and calls the MaaS gateway SSE endpoint directly, recording
4+
the arrival time of every text-delta chunk. This lets us measure per-token
5+
inter-arrival intervals independently of OpenCode's event batching.
6+
7+
Key question: does the guardrails proxy add per-token latency (reducing tok/s)
8+
or only per-request latency (increasing TTFT, no effect on tok/s)?
9+
10+
If inter-token intervals are the same with/without guardrails → only TTFT is
11+
affected (colleague's hypothesis is correct).
12+
13+
Usage:
14+
# With guardrails enabled:
15+
pytest tests/e2e/performance/test_inter_token_latency.py -v -s
16+
17+
# After toggling guardrails off, run again and compare the printed output.
18+
19+
Credentials are read from dev-user1's DevWorkspace env (OPENAI_BASE_URL,
20+
OPENAI_API_KEY) — same source OpenCode uses.
21+
"""
22+
23+
from __future__ import annotations
24+
25+
import json
26+
import random
27+
import statistics
28+
import time
29+
30+
import httpx
31+
import pytest
32+
33+
from pca_e2e import oc
34+
from pca_perf.config import user_namespace
35+
36+
MAX_TOKENS = 200
37+
RUNS = 5
38+
39+
40+
def _get_gateway_creds(ns: str) -> tuple[str, str, str]:
41+
"""Return (base_url, api_key, model_id) from dev-user1's DevWorkspace spec."""
42+
dw = oc.find_opencode_devworkspace(ns)
43+
if dw is None:
44+
pytest.skip(f"No OpenCode DevWorkspace found in {ns}")
45+
env = oc.devworkspace_env(dw)
46+
base_url = env.get("OPENAI_BASE_URL", "").rstrip("/")
47+
model_id = env.get("VLLM_MODEL_ID") or env.get("OPENAI_MODEL", "")
48+
api_key = oc.secret_data("pca-maas-apikey", "api_key", ns).strip()
49+
if not base_url or not api_key or not model_id:
50+
pytest.skip(f"Gateway creds or model_id missing in {ns}")
51+
# OPENAI_BASE_URL already ends with /v1; build full endpoint
52+
endpoint = base_url.rstrip("/") + "/chat/completions"
53+
return endpoint, api_key, model_id
54+
55+
56+
def _run_once(endpoint: str, api_key: str, model_id: str) -> dict:
57+
"""Stream one completion and return timing breakdown."""
58+
seed = random.randint(100_000, 999_999)
59+
body = {
60+
"model": model_id,
61+
"messages": [
62+
{
63+
"role": "user",
64+
"content": (
65+
f"[run-id:{seed}] Write a Python function called `add` that "
66+
"takes two numbers and returns their sum. Include a docstring. "
67+
"Do not add any other explanation."
68+
),
69+
}
70+
],
71+
"max_tokens": MAX_TOKENS,
72+
"stream": True,
73+
"stream_options": {"include_usage": True},
74+
"temperature": 0,
75+
"chat_template_kwargs": {"enable_thinking": False},
76+
}
77+
headers = {
78+
"Authorization": f"Bearer {api_key}",
79+
"Content-Type": "application/json",
80+
}
81+
82+
chunk_times: list[float] = []
83+
t_send = time.perf_counter()
84+
completion_tokens = 0
85+
86+
with httpx.stream(
87+
"POST",
88+
endpoint,
89+
json=body,
90+
headers=headers,
91+
timeout=120,
92+
) as resp:
93+
resp.raise_for_status()
94+
for line in resp.iter_lines():
95+
if not line.startswith("data:"):
96+
continue
97+
raw = line[5:].strip()
98+
if not raw or raw == "[DONE]":
99+
continue
100+
try:
101+
chunk = json.loads(raw)
102+
except json.JSONDecodeError:
103+
continue
104+
choices = chunk.get("choices") or []
105+
if choices:
106+
delta = (choices[0].get("delta") or {})
107+
if delta.get("content"):
108+
chunk_times.append(time.perf_counter())
109+
usage = chunk.get("usage") or {}
110+
if usage.get("completion_tokens"):
111+
completion_tokens = int(usage["completion_tokens"])
112+
113+
if len(chunk_times) < 2:
114+
return {}
115+
116+
ttft_ms = (chunk_times[0] - t_send) * 1000
117+
intervals_ms = [
118+
(chunk_times[i + 1] - chunk_times[i]) * 1000
119+
for i in range(len(chunk_times) - 1)
120+
]
121+
total_stream_secs = chunk_times[-1] - chunk_times[0]
122+
tok_s = len(chunk_times) / total_stream_secs if total_stream_secs > 0 else 0
123+
124+
return {
125+
"seed": seed,
126+
"chunks": len(chunk_times),
127+
"completion_tokens": completion_tokens,
128+
"ttft_ms": ttft_ms,
129+
"avg_interval_ms": statistics.mean(intervals_ms),
130+
"p50_interval_ms": statistics.median(intervals_ms),
131+
"p95_interval_ms": statistics.quantiles(intervals_ms, n=20)[-1] if len(intervals_ms) >= 20 else max(intervals_ms),
132+
"tok_s": tok_s,
133+
}
134+
135+
136+
def _print_run(i: int, r: dict) -> None:
137+
print(
138+
f" run {i:2d} seed={r['seed']} chunks={r['chunks']:3d} "
139+
f"TTFT={r['ttft_ms']:6.0f}ms "
140+
f"avg_interval={r['avg_interval_ms']:5.1f}ms "
141+
f"p50={r['p50_interval_ms']:5.1f}ms "
142+
f"tok/s={r['tok_s']:5.1f}"
143+
)
144+
145+
146+
def _print_summary(label: str, results: list[dict]) -> None:
147+
if not results:
148+
return
149+
print(f"\n{'='*70}")
150+
print(f" {label} ({len(results)} runs)")
151+
print(f"{'='*70}")
152+
for key, unit in [
153+
("ttft_ms", "ms"),
154+
("avg_interval_ms", "ms"),
155+
("p50_interval_ms", "ms"),
156+
("tok_s", "tok/s"),
157+
]:
158+
vals = [r[key] for r in results]
159+
mean = statistics.mean(vals)
160+
stdev = statistics.stdev(vals) if len(vals) > 1 else 0.0
161+
print(f" {key:22s}: mean={mean:7.2f}{unit} stdev={stdev:6.2f}{unit}")
162+
163+
164+
@pytest.mark.performance
165+
def test_inter_token_latency(oc_user: str) -> None: # noqa: ARG001
166+
ns = user_namespace(1)
167+
endpoint, api_key, model_id = _get_gateway_creds(ns)
168+
print(f"\nEndpoint: {endpoint}")
169+
print(f"Model: {model_id} max_tokens={MAX_TOKENS} runs={RUNS}\n")
170+
171+
results: list[dict] = []
172+
for i in range(1, RUNS + 1):
173+
r = _run_once(endpoint, api_key, model_id)
174+
if r:
175+
_print_run(i, r)
176+
results.append(r)
177+
time.sleep(1)
178+
179+
_print_summary("RESULTS", results)
180+
assert results, "No successful runs"

0 commit comments

Comments
 (0)