Skip to content

[benchmark] Add ITPS #8105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EmmonsCurse merged 1 commit into from
Aug 3, 2026
Merged

[benchmark] Add ITPS #8105

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions benchmarks/benchmark_serving.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ class BenchmarkMetrics:
total_output: int
request_throughput: float
request_goodput: float
input_throughput: float
output_throughput: float
total_token_throughput: float
# 解码速度(过滤 TPOT<1ms 的请求后)
Expand Down Expand Up @@ -319,6 +320,7 @@ def calculate_metrics(
total_output=sum(actual_output_lens),
request_throughput=completed / dur_s,
request_goodput=good_completed / dur_s,
input_throughput=total_input / dur_s,
output_throughput=sum(actual_output_lens) / dur_s,
total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
s_decode_filtered_mean=float(np.mean(reliable_s_decodes)) if reliable_s_decodes else 0.0,
Expand Down Expand Up @@ -762,6 +764,7 @@ async def limited_request_func_per_ip(req_input, semaphore, pbar):
print("{:<40} {:<10.3f}".format("Request throughput (req/s):", metrics.request_throughput))
if goodput_config_dict:
print("{:<40} {:<10.2f}".format("Request goodput (req/s):", metrics.request_goodput))
print("{:<40} {:<10.2f}".format("Input token throughput (tok/s):", metrics.input_throughput))
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):", metrics.output_throughput))
print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput))

Expand All @@ -772,6 +775,7 @@ async def limited_request_func_per_ip(req_input, semaphore, pbar):
"total_output_tokens": metrics.total_output,
"request_throughput": metrics.request_throughput,
"request_goodput:": (metrics.request_goodput if goodput_config_dict else None),
"input_throughput": metrics.input_throughput,
"output_throughput": metrics.output_throughput,
"reasoning_lens": [output.reasoning_tokens for output in outputs],
"total_token_throughput": metrics.total_token_throughput,
Expand Down Expand Up @@ -1045,6 +1049,7 @@ def benchmark_metrics(
print("{:<40} {:<10.2f}".format("Request throughput (req/s):", metrics.request_throughput))
if goodput_config_dict:
print("{:<40} {:<10.2f}".format("Request goodput (req/s):", metrics.request_goodput))
print("{:<40} {:<10.2f}".format("Input token throughput (tok/s):", metrics.input_throughput))
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):", metrics.output_throughput))
print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput))

Expand All @@ -1055,6 +1060,7 @@ def benchmark_metrics(
"total_output_tokens": metrics.total_output,
"request_throughput": metrics.request_throughput,
"request_goodput:": (metrics.request_goodput if goodput_config_dict else None),
"input_throughput": metrics.input_throughput,
"output_throughput": metrics.output_throughput,
"total_token_throughput": metrics.total_token_throughput,
"input_lens": [output.prompt_len for output in outputs],
Expand Down
Loading