diff --git a/docs/en/advanced_guides/custom_dataset/clip.md b/docs/en/advanced_guides/custom_dataset/clip.md index 1fdfe8a96..b1209d3e9 100644 --- a/docs/en/advanced_guides/custom_dataset/clip.md +++ b/docs/en/advanced_guides/custom_dataset/clip.md @@ -46,7 +46,7 @@ task_cfg = { } ``` ```{seealso} -[Full Parameter Explanation](../../user_guides/backend/rageval_backend/clip_benchmark.md#configure-evaluation-parameters) +[Full Parameter Reference](../../user_guides/backend/rageval_backend/clip_benchmark.md#full-parameter-reference) ``` Where: diff --git a/docs/en/advanced_guides/custom_dataset/embedding.md b/docs/en/advanced_guides/custom_dataset/embedding.md index cb1a7848a..7d0b09357 100644 --- a/docs/en/advanced_guides/custom_dataset/embedding.md +++ b/docs/en/advanced_guides/custom_dataset/embedding.md @@ -148,7 +148,7 @@ Each task in the `custom_tasks` list corresponds to a `CustomTaskConfig` with th | `eval_splits` | List[str] | `["test"]` | Evaluation splits list | ```{note} -Other evaluation parameters (such as `models`, `limits`, `overwrite_results`, etc.) are consistent with the default configuration. See [MTEB Evaluation Parameter Reference](../../user_guides/backend/rageval_backend/mteb.md#parameter-explanation) for details. +Other evaluation parameters (such as `models`, `limits`, `overwrite_results`, etc.) are consistent with the default configuration. See [MTEB Evaluation Parameter Reference](../../user_guides/backend/rageval_backend/mteb.md#full-parameter-reference) for details. ``` ## FAQ diff --git a/docs/en/user_guides/stress_test/examples.md b/docs/en/user_guides/stress_test/examples.md index 4c2aff516..7914406fb 100644 --- a/docs/en/user_guides/stress_test/examples.md +++ b/docs/en/user_guides/stress_test/examples.md @@ -1,140 +1,70 @@ # Examples -## Using Local Model Inference +This page walks through copy-pasteable commands in the order of *target → input data → request parameters → load pattern → observing results*. For the full parameter reference see [Parameters](./parameters.md); for multi-turn scenarios see [Multi-turn Conversation Benchmark](./multi_turn.md). -This project supports inference using local transformers and vllm (vllm needs to be installed first). The `--model` can be filled with a modelscope model name, such as `Qwen/Qwen2.5-0.5B-Instruct`; or you can directly specify the model weight path, such as `/path/to/model_weights`, without needing to specify the `--url` parameter. +## Local Model Inference -**Inference using transformers** +Both local transformers inference and vLLM inference (vllm must be installed first) are supported, and neither needs `--url`. `--model` accepts a ModelScope model name such as `Qwen/Qwen2.5-0.5B-Instruct`, or a direct path to model weights such as `/path/to/model_weights`. -```bash -evalscope perf \ - --model 'Qwen/Qwen2.5-0.5B-Instruct' \ - --attn-implementation flash_attention_2 \ # Optional, or choose from [flash_attention_2|eager|sdpa] - --number 20 \ - --parallel 2 \ - --api local \ - --dataset openqa -``` +**transformers inference**: pass `--api local`. -**Inference using vllm** ```bash evalscope perf \ - --model 'Qwen/Qwen2.5-0.5B-Instruct' \ - --number 20 \ - --parallel 2 \ - --api local_vllm \ - --dataset openqa + --model 'Qwen/Qwen2.5-0.5B-Instruct' \ + --number 20 \ + --parallel 2 \ + --api local \ + --dataset openqa ``` -## Using `prompt` -```bash -evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --number 20 \ - --api openai \ - --temperature 0.9 \ - --max-tokens 1024 \ - --prompt 'Write a science fiction story, please begin your performance' -``` -You can also use a local file as a prompt: +Optionally add `--attn-implementation`, choosing from `flash_attention_2`, `eager`, or `sdpa`. + +**vLLM inference**: pass `--api local_vllm`. + ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --number 20 \ - --api openai \ - --temperature 0.9 \ - --max-tokens 1024 \ - --prompt @prompt.txt + --model 'Qwen/Qwen2.5-0.5B-Instruct' \ + --number 20 \ + --parallel 2 \ + --api local_vllm \ + --dataset openqa ``` -## Complex Requests -Using `stop`, `stream`, `temperature`, etc.: +## Input Construction -```bash -evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --read-timeout 120 \ - --connect-timeout 120 \ - --number 20 \ - --max-prompt-length 128000 \ - --min-prompt-length 128 \ - --api openai \ - --temperature 0.7 \ - --max-tokens 1024 \ - --stop '<|im_end|>' \ - --dataset openqa \ - --stream -``` - -## Using `query-template` - -You can set request parameters in the `query-template`: +### Fixed Prompt + +Use `--prompt` to send one fixed prompt for every request, with no dataset involved. ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --read-timeout 120 \ - --connect-timeout 120 \ - --number 20 \ - --max-prompt-length 128000 \ - --min-prompt-length 128 \ - --api openai \ - --query-template '{"model": "%m", "messages": [{"role": "user","content": "%p"}], "stream": true, "skip_special_tokens": false, "stop": ["<|im_end|>"], "temperature": 0.7, "max_tokens": 1024}' \ - --dataset openqa -``` -Where `%m` and `%p` will be replaced by the model name and the prompt. - -You can set request parameters in the query-template: + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --number 20 \ + --api openai \ + --temperature 0.9 \ + --max-tokens 1024 \ + --prompt 'Write a science fiction story, please begin your performance' +``` -```{code-block} json -:caption: template.json +You can also read it from a local file with the `@` prefix: -{ - "model":"%m", - "messages":[ - { - "role":"user", - "content":"%p" - } - ], - "stream":true, - "skip_special_tokens":false, - "stop":[ - "<|im_end|>" - ], - "temperature":0.7, - "max_tokens":1024 -} -``` ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --read-timeout 120 \ - --connect-timeout 120 \ - --number 20 \ - --max-prompt-length 128000 \ - --min-prompt-length 128 \ - --api openai \ - --query-template @template.json \ - --dataset openqa -``` - -## Using the Random Dataset + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --number 20 \ + --api openai \ + --temperature 0.9 \ + --max-tokens 1024 \ + --prompt @prompt.txt +``` + +### Random Dataset Randomly generate prompts based on `prefix-length`, `max-prompt-length`, and `min-prompt-length`. It is necessary to specify `tokenizer-path`. The number of tokens in the generated prompt is uniformly distributed between `prefix_length + min-prompt-length` and `prefix_length + max-prompt-length`. In a single test, all requests have the same prefix portion. @@ -142,8 +72,6 @@ Randomly generate prompts based on `prefix-length`, `max-prompt-length`, and `mi Due to the influence of chat_template and tokenization algorithms, there may be some discrepancies in the number of tokens in the generated prompts, and it is not an exact specified token count. ``` -Execute the following command: - ```bash evalscope perf \ --parallel 20 \ @@ -167,7 +95,7 @@ To ensure the server receives exactly the configured number of tokens, add `--to The server will receive exactly `prefix_length + inner_seq_length` tokens, which falls within `[min-prompt-length, max-prompt-length]`. Compatible with vLLM, SGLang, LMDeploy, and other frameworks that accept token-ID input; not supported for the `random_vl` dataset. ``` -## Using the Random Multimodal Dataset +### Random Multimodal Dataset Use the `random_vl` dataset to randomly generate image and text inputs. Based on the `random` dataset, it adds image-related parameters (`image-width`, `image-height`, `image-format`, `image-num`). @@ -192,56 +120,147 @@ evalscope perf \ --debug ``` -## Embedding Model Stress Testing +### Long-context Prefix Injection + +To benchmark 128K/256K-scale long contexts with **real text**, the `random` dataset gives you a fixed length but only high-entropy meaningless tokens, which cannot reproduce real characteristics such as prefix-cache hit rates or MTP acceptance rates. Real instruction datasets, on the other hand, are mostly 4K-8K tokens, so setting `target_input_len` to 128K either filters everything out or leaves lengths uneven. -Use `openai_embedding` API mode and `random_embedding` dataset for stress testing. When using the random dataset, you need to specify `tokenizer-path` to generate query of specified length. +`prefix_file` solves this: point it at a long text corpus (a book, documentation, a code base, etc.) and the framework slices it to exactly `target_input_len − prompt length` tokens and prepends it to each short prompt, so **every request's input hits the target length** while keeping the low-entropy character of real human language. See [Long-context Prefix Injection](./parameters.md#long-context-prefix-injection) for the parameter reference. + +**Inject the prefix as the system role** (recommended, matching real RAG traffic): ```bash evalscope perf \ - --parallel 2 \ - --number 10 \ - --model 'text-embedding-v4' \ - --url 'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings' \ - --api-key ${DASHSCOPE_API_KEY} \ - --api openai_embedding \ - --dataset random_embedding \ - --min-prompt-length 256 \ - --max-prompt-length 256 \ - --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' + --parallel 4 \ + --model Qwen2.5-0.5B-Instruct \ + --url http://127.0.0.1:8801/v1/chat/completions \ + --api openai \ + --dataset openqa \ + --tokenizer-path Qwen/Qwen2.5-0.5B-Instruct \ + --dataset-args '{"target_input_len": 8192, "prefix_file": "long_text.txt", "prefix_role": "system"}' \ + --max-tokens 128 \ + --number 20 ``` -## Rerank Model Stress Testing +Each request is built as `[{"role": "system", "content": ""}, {"role": "user", "content": ""}]`, and the `Input Tokens` reported by the server are identical across requests (`avg` = `p50` = `p99` = `max`), which makes it easy to benchmark a single length point. -Use `openai_rerank` API mode and `random_rerank` dataset for stress testing. When using the random dataset, you need to specify `tokenizer-path` to generate query of specified length. +**Prepend the prefix to the user message**: set `prefix_role` to `user` and the prefix is prepended directly to the user question, producing a single user message. Useful when you do not want to introduce a system role. -You can specify data generation parameters through `extra-args`: -- `num_documents`: Number of documents per query -- `document_length_ratio`: Document length multiplier relative to query length +```bash +evalscope perf \ + --parallel 2 \ + --model Qwen2.5-0.5B-Instruct \ + --url http://127.0.0.1:8801/v1/chat/completions \ + --api openai \ + --dataset openqa \ + --tokenizer-path Qwen/Qwen2.5-0.5B-Instruct \ + --dataset-args '{"target_input_len": 131072, "prefix_file": "long_text.txt", "prefix_role": "user"}' \ + --number 10 +``` + +```{note} +**Practical notes** + +- **Preparing the prefix corpus**: any UTF-8 plain-text file works. When the corpus has fewer tokens than the budget it is automatically repeated (tiled) with a warning — to avoid repetition entirely, use a corpus with more tokens than `target_input_len`. +- **Stay under the server limit**: `target_input_len` must be smaller than the server's `max_model_len` (adjustable via `--max-model-len` in vLLM), otherwise requests are rejected. Note that the chat template itself also consumes a dozen or so extra tokens. +- **Supported datasets**: `openqa`, `longalpaca`, `line_by_line` (plain-text lines only), and ShareGPT (`share_gpt_zh` / `share_gpt_en`); all require `--tokenizer-path`. ShareGPT budgets against the whole conversation, so a conversation already longer than `target_input_len` is skipped entirely. +- **Not compatible with `drop`**: `prefix_file` and `input_len_mode="drop"` are mutually exclusive (`drop` only keeps prompts that already fill the target, leaving a zero prefix budget) and the combination is rejected at config time; use the default `cap` and let the prefix fill the rest. +- **Prefix-cache effect**: all requests share the same prefix head, so re-running the same configuration shows a clear TTFT drop (the engine hits its prefix cache). Conversely, to measure **cache-free** cold-start performance, use the `random` dataset instead (its `--dataset-offset` mechanism keeps prompts distinct across runs). +- **`/v1/completions` endpoint**: this endpoint does not apply a chat template, so `prefix_role="system"` automatically falls back to plain-text concatenation with a warning; the prefix and prompt then sit directly adjacent and tokens may merge or split across the join, making the measured total differ from the target by about ±1 token (the same applies to `prefix_role="user"`; see "Join boundary" in the [parameter reference](./parameters.md#long-context-prefix-injection)). +``` + +## Request Configuration + +### Generation Parameters and Timeouts + +Combine `stop`, `stream`, `temperature`, and read/connect timeouts: + +```bash +evalscope perf \ + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --read-timeout 120 \ + --connect-timeout 120 \ + --number 20 \ + --max-prompt-length 128000 \ + --min-prompt-length 128 \ + --api openai \ + --temperature 0.7 \ + --max-tokens 1024 \ + --stop '<|im_end|>' \ + --dataset openqa \ + --stream +``` + +### Custom Request Body + +Use `--query-template` to define the complete request-body JSON, where `%m` and `%p` are replaced by the model name and the prompt: ```bash evalscope perf \ - --parallel 2 \ - --number 10 \ - --model 'qwen3-rerank' \ - --url 'https://dashscope.aliyuncs.com/compatible-api/v1/reranks' \ - --api-key ${DASHSCOPE_API_KEY} \ - --api openai_rerank \ - --dataset random_rerank \ - --min-prompt-length 256 \ - --max-prompt-length 256 \ - --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' \ - --extra-args '{"num_documents": 5, "document_length_ratio": 3}' + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --read-timeout 120 \ + --connect-timeout 120 \ + --number 20 \ + --max-prompt-length 128000 \ + --min-prompt-length 128 \ + --api openai \ + --query-template '{"model": "%m", "messages": [{"role": "user","content": "%p"}], "stream": true, "skip_special_tokens": false, "stop": ["<|im_end|>"], "temperature": 0.7, "max_tokens": 1024}' \ + --dataset openqa ``` -## Warmup Benchmarking +For longer templates, write the JSON to a local file and reference it with the `@` prefix: + +```{code-block} json +:caption: template.json + +{ + "model":"%m", + "messages":[ + { + "role":"user", + "content":"%p" + } + ], + "stream":true, + "skip_special_tokens":false, + "stop":[ + "<|im_end|>" + ], + "temperature":0.7, + "max_tokens":1024 +} +``` + +```bash +evalscope perf \ + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --read-timeout 120 \ + --connect-timeout 120 \ + --number 20 \ + --max-prompt-length 128000 \ + --min-prompt-length 128 \ + --api openai \ + --query-template @template.json \ + --dataset openqa +``` + +## Load Patterns + +### Warmup Send a batch of warmup requests before the formal benchmark to eliminate cold-start effects (e.g. KV-cache filling, JIT compilation, connection pool initialization) and produce more accurate performance metrics. Warmup requests are sent with the same concurrency and rate as the benchmark but **excluded from performance metrics** (latency, throughput, percentiles, etc.). -**1. Absolute Count Mode** - -Specify the exact number of warmup requests: +**Absolute count mode**: set `--warmup-num` to an integer `>= 1`, the exact number of warmup requests. ```bash evalscope perf \ @@ -257,9 +276,7 @@ evalscope perf \ The above command sends 10 warmup requests first, then 100 benchmark requests. Metrics are computed only from the latter 100 requests. -**2. Ratio Mode** - -Use a float between 0 and 1 to compute warmup count as a proportion of `--number`. This is especially useful for sweep mode where each run has a different `--number`: +**Ratio mode**: set `--warmup-num` to a float between 0 and 1 to compute the warmup count as a proportion of `--number`. This is especially useful for sweep mode where each run has a different `--number`. ```bash evalscope perf \ @@ -283,7 +300,7 @@ evalscope perf \ - In multi-turn mode, `--warmup-num` specifies the number of warmup conversations (consistent with the `--number` semantics); all turns within a warmup conversation are excluded from metrics. ``` -## Open-loop Mode +### Open-loop Mode In open-loop mode, requests are dispatched immediately following a Poisson arrival schedule (controlled by `--rate`), without waiting for the server to return responses. This models realistic traffic patterns where arrivals are independent of service time. By specifying multiple rate values in a single command, you can automatically sweep the throughput-latency curve. @@ -312,13 +329,13 @@ evalscope perf \ - Core difference from closed-loop (default) mode: closed-loop workers wait for a response before sending the next request (backpressure protection); open-loop fires requests on schedule without waiting (closer to real traffic). ``` -## Production Traffic Replay (workload_trace) +### Production Traffic Replay The `workload_trace` dataset replays recorded production traffic verbatim following its **original arrival timing**, closely matching real-world load — bursty arrivals, heterogeneous request shapes, and multi-model routing that synthetic datasets (`random`, `openqa`, etc.) cannot reproduce. It builds on open-loop scheduling, but arrival times come from the trace's `timestamp` field, so **no `--rate` is needed**. -Prepare a JSONL trace file (one request record per line); field reference is in [Parameters · Case 3](./parameters.md#dataset-args-per-dataset-arguments): +Prepare a JSONL trace file (one request record per line); field reference is in [Parameters · Production Traffic Replay](./parameters.md#production-traffic-replay): -```jsonl +```json {"body": {"model": "qwen-plus", "messages": [{"role": "user", "content": "hello"}]}, "timestamp": 1700000000.0} {"body": {"model": "qwen-max", "messages": [{"role": "user", "content": "write a poem"}]}, "timestamp": 1700000001.5, "request_id": "req-42"} ``` @@ -365,10 +382,55 @@ evalscope perf \ - Use `--name` to set a meaningful output directory name (without `--model`, the directory name defaults to the dataset name). ``` +## Embedding and Rerank + +### Embedding Models + +Use `openai_embedding` API mode and the `random_embedding` dataset. When using the random dataset, you need to specify `tokenizer-path` to generate queries of the specified length. + +```bash +evalscope perf \ + --parallel 2 \ + --number 10 \ + --model 'text-embedding-v4' \ + --url 'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings' \ + --api-key ${DASHSCOPE_API_KEY} \ + --api openai_embedding \ + --dataset random_embedding \ + --min-prompt-length 256 \ + --max-prompt-length 256 \ + --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' +``` + +### Rerank Models + +Use `openai_rerank` API mode and the `random_rerank` dataset. When using the random dataset, you need to specify `tokenizer-path` to generate queries of the specified length. + +You can specify data generation parameters through `extra-args`: + +- `num_documents`: number of documents per query +- `document_length_ratio`: document length multiplier relative to query length + +```bash +evalscope perf \ + --parallel 2 \ + --number 10 \ + --model 'qwen3-rerank' \ + --url 'https://dashscope.aliyuncs.com/compatible-api/v1/reranks' \ + --api-key ${DASHSCOPE_API_KEY} \ + --api openai_rerank \ + --dataset random_rerank \ + --min-prompt-length 256 \ + --max-prompt-length 256 \ + --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' \ + --extra-args '{"num_documents": 5, "document_length_ratio": 3}' +``` + ## Debugging Requests + Use the `--debug` option to output the requests and responses. -**Non-`stream` Mode Output Example** +**Non-`stream` mode output example** ```text 2024-11-27 11:25:34,161 - evalscope - http_client.py - on_request_start - 116 - DEBUG - Starting request: )> @@ -376,7 +438,7 @@ Use the `--debug` option to output the requests and responses. 2024-11-27 11:25:38,172 - evalscope - http_client.py - on_response_chunk_received - 140 - DEBUG - Request received: ``` -**`stream` Mode Output Example** +**`stream` mode output example** ```text 2024-11-27 20:02:24,760 - evalscope - http_client.py - _handle_stream - 57 - DEBUG - Response recevied: data: {"model":"Qwen2.5-0.5B-Instruct","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"重要的"},"finish_reason":null}],"usage":null} @@ -393,14 +455,18 @@ Use the `--debug` option to output the requests and responses. 2024-11-27 20:02:25,113 - evalscope - http_client.py - _handle_stream - 57 - DEBUG - Response recevied: data: [DONE] ``` -## Visualizing Test Results +## Visualizing Results + +### WandB + +Install wandb: -### Using WandB -Please install wandb using the following command: ```bash pip install wandb ``` + Add the following parameters before starting the test: + ```bash --visualizer wandb --name 'name_of_wandb_log' @@ -408,36 +474,40 @@ Add the following parameters before starting the test: ![wandb sample](https://modelscope.oss-cn-beijing.aliyuncs.com/resource/wandb_sample.png) +### SwanLab -### Using SwanLab +Install SwanLab: -Please install SwanLab using the following command: ```bash pip install swanlab ``` Add the following parameters before starting the test: + ```bash # You can use the SWANLAB_PROJ_NAME environment variable to specify the project name --visualizer swanlab --name 'name_of_swanlab_log' -``` +``` ![swanlab sample](https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/swanlab.png) +### ClearML + +Install ClearML: -### Using ClearML -Please install ClearML using the following command: ```bash pip install clearml ``` Initialize the ClearML server: + ```bash clearml-init ``` Add the following parameters before starting the test: + ```bash # You can use the CLEARML_PROJECT_NAME environment variable to specify the project name --visualizer clearml diff --git a/docs/en/user_guides/stress_test/multi_turn.md b/docs/en/user_guides/stress_test/multi_turn.md index 016d850d4..45379df83 100644 --- a/docs/en/user_guides/stress_test/multi_turn.md +++ b/docs/en/user_guides/stress_test/multi_turn.md @@ -1,50 +1,12 @@ # Multi-turn Conversation Benchmark -The multi-turn conversation benchmark allows you to test a model service in realistic multi-turn interaction scenarios. Unlike a standard benchmark, multi-turn mode appends the model's **actual replies** to the conversation context so that every subsequent request carries the full history — faithfully simulating a user's continuous conversation with the model, and enabling measurement of latency, throughput, and KV-cache utilization as context grows. +The multi-turn conversation benchmark tests a model service in realistic multi-turn interaction scenarios: after each successful turn, the model's **actual reply** is appended to the context, and the next request carries the full history rather than just the current user message. This makes it possible to measure latency and throughput as context grows. -## Features +Based on client-side token counts, the framework also estimates the proportion of history tokens relative to the total input tokens in each request — i.e. the theoretical upper bound of tokens that could benefit from server-side prefix caching. Whether caching actually occurs depends on whether the server has prefix caching enabled and has retained the relevant cache. -- **Real context accumulation**: After each successful turn, the model's actual output is appended to the conversation history; the next turn sends the complete history rather than just the current user message. -- **Approx KV cache hit rate estimation**: Based on client-side token counts, estimates the proportion of history tokens relative to the total input tokens in each request — i.e., the theoretical upper bound of tokens that could benefit from server-side prefix caching. Whether caching actually occurs depends on whether the server has prefix caching enabled and has retained the relevant cache. -- **Multiple dataset support**: Provides random synthetic (`random_multi_turn`), real conversations (`share_gpt_zh_multi_turn` / `share_gpt_en_multi_turn`), custom local data (`custom_multi_turn`), real Agent trajectories (`swe_smith`), and production agentic trace replay (`trie_agentic_coding` / `trie_code_qa` / `trie_office_work`) datasets. -- **Consistent parameter semantics**: `--number` is the total number of conversations and `--parallel` is the number of concurrent conversations, keeping the same semantics as standard benchmark mode. +`--number` is the total number of conversations and `--parallel` is the number of concurrent conversations, keeping the same semantics as single-turn benchmarks. -## Parameters - -### Multi-turn Specific Parameters - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `--multi-turn` | `bool` | Enable multi-turn conversation benchmark mode | `False` | -| `--min-turns` | `int` | Minimum number of user turns per conversation; used by `random_multi_turn` only | `1` | -| `--max-turns` | `int` | Maximum number of user turns per conversation; **required** for `random_multi_turn`; optional for ShareGPT / `custom_multi_turn` datasets to truncate long conversations; for `swe_smith` live construction, the per-conversation turn count is sampled from `[min_turns, max_turns]` | `None` | -| `--dataset-offset` | `int` | Skip the first N conversations in the dataset; useful for sharded testing or avoiding cache hits | `0` | - -### `multi_turn_args` (swe_smith-specific parameters) - -```{note} -`--multi-turn-args` is **deprecated**; pass the same parameters via the unified `--dataset-args` instead. The old flag still works and its content is automatically folded into `--dataset-args` (on a key conflict, `--dataset-args` takes precedence). The parameter key names below are unchanged. -``` - -The `swe_smith` dataset's live construction mode supports fine-grained control of conversation structure and token-length targets via `--dataset-args`. - -The number of turns per conversation is sampled from `[--min-turns, --max-turns]`; the amount of content filled per turn is controlled by the token-length parameters below. - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `first_turn_length` | `int` | Target prompt token count for turn 1; trajectory messages are sliced until this length is reached | `65000` | -| `subsequent_turn_length` | `int` | Target token increment per subsequent turn; controls the delta size added each round | `500` | -| `chars_per_token` | `float` | Characters-per-token estimate used for pre-filtering trajectories when no tokenizer is available | `3.0` | -| `num_workers` | `int` | **Deprecated**. Use top-level `--num-workers` instead. Previously controlled parallel workers for live conversation building. | `4` | - -### Semantics of Existing Parameters in Multi-turn Mode - -| Parameter | Meaning in multi-turn mode | -|-----------|---------------------------| -| `--number` | Total number of **conversations** to run; all workers stop once this many conversations have been completed | -| `--parallel` | Number of concurrently active conversations (each worker owns one conversation) | - -## Workflow +## How It Works 1. **Load conversation pool**: At startup, conversations are read sequentially from the dataset file and pre-loaded into memory, up to a maximum of `--number` conversations (to avoid excessive memory usage with large datasets). @@ -91,11 +53,46 @@ The number of turns per conversation is sampled from `[--min-turns, --max-turns] 7. **Metrics aggregation**: After all workers finish, all latency, throughput, and multi-turn-specific metrics (average context turns, KV cache hit rate) are aggregated and output. -> **Note**: When the request success rate is below 100%, interrupted conversations do not contribute subsequent turns to the context, which may result in lower reported KV cache hit rates. +```{note} +When the request success rate is below 100%, interrupted conversations do not contribute subsequent turns to the context, which may result in lower reported KV cache hit rates. +``` + +## Parameters + +### Multi-turn Specific Parameters + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `--multi-turn` | `bool` | Enable multi-turn conversation benchmark mode | `False` | +| `--min-turns` | `int` | Minimum number of user turns per conversation; used by `random_multi_turn` only | `1` | +| `--max-turns` | `int` | Maximum number of user turns per conversation; **required** for `random_multi_turn`; optional for ShareGPT / `custom_multi_turn` datasets to truncate long conversations; for `swe_smith` live construction, the per-conversation turn count is sampled from `[min_turns, max_turns]` | `None` | +| `--dataset-offset` | `int` | Skip the first N conversations in the dataset; useful for sharded testing or avoiding cache hits | `0` | + +### Multi-turn Semantics of Common Parameters + +The following parameters mean different things in single-turn and multi-turn mode; in multi-turn mode they are all counted in **conversations**: + +| Parameter | Meaning in multi-turn mode | +|-----------|---------------------------| +| `--number` | Total number of **conversations** to run; all workers stop once this many conversations have been completed | +| `--parallel` | Number of concurrently active conversations (each worker owns one conversation) | +| `--warmup-num` | Number of warmup **conversations**; all turns within a warmup conversation are excluded from metrics | +| `--duration` | **Soft exit**: once the deadline elapses no new conversations are claimed, but already in-flight conversations run every remaining turn before exit | +| `--tokenize-prompt` | **Not supported**; silently ignored. Multi-turn conversations are always sent as message dicts to `/v1/chat/completions` | + +The soft-exit semantics of `--duration` keep every conversation complete, so partial conversations don't skew statistics. Three cap combinations: + +| Command | Meaning | +|---|---| +| `--number 50` | run 50 conversations, no time limit | +| `--duration 300` | run for 300 seconds, however many conversations fit | +| `--number 50 --duration 300` | both caps; **whichever is reached first** ends the run (recommended) | + +Side effect: actual wall_time may overshoot `--duration` slightly — the overshoot equals the time needed for in-flight conversations to finish. ## Datasets -evalscope provides the following multi-turn datasets: +evalscope provides the following multi-turn datasets: synthetic data (`random_multi_turn`), real conversations (`share_gpt_*_multi_turn`), custom local data (`custom_multi_turn`), real Agent trajectories (`swe_smith`), and production agentic trace replay (`trie_*`). ### random_multi_turn @@ -114,9 +111,7 @@ Each conversation produced by the dataset has the following structure: ] ``` -> **Note**: `--tokenize-prompt` is not supported in multi-turn mode and will be silently ignored. Multi-turn conversations are always sent as message dicts to the `/v1/chat/completions` endpoint. - -**Usage example**: Quickly evaluate service performance at a specified prompt length distribution and conversation depth without a real dataset. +**When to use**: Quickly evaluate service performance at a specified prompt length distribution and conversation depth without a real dataset. ```bash evalscope perf \ @@ -189,9 +184,11 @@ Runtime context structure (when sending turn 2): ] ``` -> **Note**: The reference assistant replies in the dataset are included for structural completeness only and are never sent directly to the model. At runtime, workers always append the model's **actual output** to the context to ensure accurate history. +```{note} +The reference assistant replies in the dataset are included for structural completeness only and are never sent directly to the model. At runtime, workers always append the model's **actual output** to the context to ensure accurate history. +``` -**Usage example**: Evaluate service performance using a realistic user conversation distribution, better reflecting production environment behavior. +**When to use**: Evaluate service performance using a realistic user conversation distribution, better reflecting production environment behavior. ```bash evalscope perf \ @@ -267,6 +264,7 @@ Uses a local JSONL file as a custom multi-turn conversation dataset. Each line s ``` Each line must satisfy: + - Must be a JSON array. - Every element must have `role` and `content` fields. - `role` must be either `user` or `assistant`. @@ -282,18 +280,11 @@ Runtime context structure (when sending turn 2): ] ``` -> **Note**: The `assistant` messages in the dataset are used only to identify conversation structure and are **never** sent directly to the model. At runtime, workers always append the model's actual output to the context to ensure accurate history. - -**Usage example**: You have conversation data already in OpenAI messages format and want to benchmark directly without any format conversion. - -First, prepare the JSONL data file (one conversation per line): - -```json -[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi! How can I help you?"}, {"role": "user", "content": "Write me a poem"}, {"role": "assistant", "content": "Sure, ..."}, {"role": "user", "content": "Write another one"}] -[{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "Paris."}, {"role": "user", "content": "What are some famous landmarks there?"}] +```{note} +The `assistant` messages in the dataset are used only to identify conversation structure and are **never** sent directly to the model. At runtime, workers always append the model's actual output to the context to ensure accurate history. ``` -Then run the benchmark: +**When to use**: You have conversation data already in OpenAI messages format and want to benchmark directly without any format conversion. ```bash evalscope perf \ @@ -318,16 +309,26 @@ Two data source modes are supported: - **Pre-built JSON mode** (recommended): Specify `--dataset-path` to load a pre-generated `agentic_dataset.json`. No tokenizer is required and startup is fast. - **Live construction mode** (no `--dataset-path`): Pulls raw trajectories from ModelScope at runtime and dynamically builds conversations. `--tokenizer-path` is **required** for accurate token counting. -Common features of both modes: -- **Offset support**: Skip the first N conversations via `--dataset-offset`, useful for sharded testing or avoiding KV cache hot-spots. +Both modes support `--dataset-offset` to skip the first N conversations, useful for sharded testing or avoiding KV cache hot-spots. + +**Length Control Parameters** + +In live construction mode, the number of turns per conversation is sampled from `[--min-turns, --max-turns]`, and each turn is filled according to the token-length parameters below. Pass them via `--dataset-args`: -> **Note**: The turn count for each conversation is sampled from `[--min-turns, --max-turns]`; the amount of content per turn is determined by `first_turn_length` / `subsequent_turn_length`. +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `first_turn_length` | `int` or `[int, int]` | Target prompt token count for turn 1; trajectory messages are sliced until this length is reached
• A single integer: fixed value, e.g. `65000`
• A two-element list: `[min, max]`, sampled uniformly at random per conversation, e.g. `[32000, 65000]` | `65000` | +| `subsequent_turn_length` | `int` or `[int, int]` | Target token increment per subsequent turn; controls the delta size added each round
• A single integer: fixed value
• A two-element list: `[min, max]`, sampled uniformly at random per conversation | `500` | +| `chars_per_token` | `float` | Characters-per-token estimate used for pre-filtering trajectories when no tokenizer is available | `3.0` | +| `num_workers` | `int` | **Deprecated**. Use top-level `--num-workers` instead. Previously controlled parallel workers for live conversation building. | `4` | -**Building the Dataset** +```{note} +`--multi-turn-args` is **deprecated**; pass the parameters above via the unified `--dataset-args` instead. The old flag still works and its content is automatically folded into `--dataset-args` (on a key conflict, `--dataset-args` takes precedence). The parameter key names are unchanged. +``` -It is recommended to pre-build `agentic_dataset.json` using `examples/perf/build_swe_smith_dataset.py` before running a benchmark — build once, reuse many times, avoiding repeated downloads and on-the-fly construction. +**Pre-building the Dataset** -**Key parameters**: +It is recommended to pre-build `agentic_dataset.json` using `examples/perf/build_swe_smith_dataset.py` before running a benchmark — build once, reuse many times, avoiding repeated downloads and on-the-fly construction. Note that the script takes CLI dashed flags, which map one-to-one onto the underscored `--dataset-args` keys above: | Parameter | Description | Default | |-----------|-------------|--------| @@ -349,7 +350,7 @@ python examples/perf/build_swe_smith_dataset.py \ --min-turns 3 \ --max-turns 8 \ --number 128 \ - --output-path agentic_dataset.json \ + --output-path outputs/agentic_dataset.json \ --seed 42 \ --num-workers 8 ``` @@ -372,8 +373,6 @@ evalscope perf \ --parallel 20 ``` -> **Note**: `--dataset-offset` skips the first N conversations in the dataset, making it suitable for multi-machine sharded benchmarking or avoiding KV cache hot-spots. - **Usage Example: Live Construction Mode** Automatically pulls SWE-smith-trajectories from ModelScope and constructs conversations at runtime. `--tokenizer-path` is required: @@ -385,12 +384,12 @@ evalscope perf \ --api openai \ --dataset swe_smith \ --tokenizer-path YOUR_MODEL \ - --max-tokens 512 \ + --max-tokens 512 1024 \ --min-tokens 512 \ --multi-turn \ --dataset-args '{ - "first_turn_length": 8192, - "subsequent_turn_length": 1024 + "first_turn_length": [4096, 8192], + "subsequent_turn_length": [512, 1024] }' \ --min-turns 3 \ --max-turns 8 \ @@ -424,8 +423,6 @@ evalscope perf ... --dataset trie_office_work --dataset-path /path/to/your_trace **Required**: `--tokenizer-path` (for synthesizing prompts to exact target lengths) -**Usage example**: - ```bash evalscope perf \ --model qwen-plus \ @@ -442,11 +439,13 @@ evalscope perf \ --stream ``` -> **Note**: trie trace replay relies on `ignore_eos` to force the model to generate to the exact recorded length. vLLM and SGLang support this parameter; DashScope / OpenAI API and most cloud services do not — the model stops at natural EOS, so actual output will be shorter than the recorded length. This does not affect latency / cache hit / decode TPS correctness. +```{note} +trie trace replay relies on `ignore_eos` to force the model to generate to the exact recorded length. vLLM and SGLang support this parameter; DashScope / OpenAI API and most cloud services do not — the model stops at natural EOS, so actual output will be shorter than the recorded length. This does not affect latency / cache hit / decode TPS correctness. +``` -## Multi-turn Output Metrics +## Output Metrics -In addition to the Performance Overview and Per-Request Metrics tables, multi-turn mode outputs the following additional tables. +In addition to the Performance Overview and Per-Request Metrics tables, multi-turn mode outputs the following. ### Per-Trace Metrics @@ -463,6 +462,10 @@ Per-conversation (trace) metrics, aggregated as mean / p50 / p90 / p99 / max acr Also outputs `trace_summary.json` to the results directory. +```{note} +When using the `/v1/chat/completions` endpoint, the chat template adds 10-50 extra tokens per turn (role markers, special tokens, etc.), causing Cache Hit Rate to be 2-3pp lower than raw completions. This is expected behaviour. +``` + ### Workload Throughput Time-based token throughput rates, output in all modes (single-turn and multi-turn): @@ -495,29 +498,7 @@ Steady-state is the fairest number for "how fast can this endpoint sustainably r ### First-Turn vs Subsequent-Turn TTFT -The `TTFT (ms)` in the Per-Request Metrics table averages every turn. In multi-turn scenarios, the first turn is a cold prefill while subsequent turns benefit heavily from prefix cache, so the two TTFT types can differ by 2-10x. The Per-Request Metrics table shows: +The `TTFT (ms)` in the Per-Request Metrics table averages every turn. In multi-turn scenarios, the first turn is a cold prefill while subsequent turns benefit heavily from prefix cache, so the two TTFT types can differ by 2-10x. The Per-Request Metrics table therefore shows two extra rows, which appear only in multi-turn runs: - `First-Turn TTFT (ms)` - `Subsequent-Turn TTFT (ms)` - -These appear only in multi-turn runs; single-turn benchmarks omit them. - -## `--duration` Soft Exit - -In multi-turn mode, `--duration` uses soft-exit semantics: once the deadline elapses, **no new conversations are claimed**, but **already in-flight conversations run every remaining turn** before exit. This keeps every conversation complete, so partial conversations don't skew statistics. - -Three cap combinations: - -| Command | Meaning | -|---|---| -| `--number 50` | run 50 conversations, no time limit | -| `--duration 300` | run for 300 seconds, however many conversations fit | -| `--number 50 --duration 300` | both caps; **whichever is reached first** ends the run (recommended) | - -Side effect: actual wall_time may overshoot `--duration` slightly (the overshoot equals the time needed for in-flight conversations to finish). - -## FAQs - -### Chat-template token overhead - -When using the `/v1/chat/completions` endpoint, the chat template adds 10-50 extra tokens per turn (role markers, special tokens, etc.), causing Cache Hit Rate to be 2-3pp lower than raw completions. This is expected behaviour. diff --git a/docs/en/user_guides/stress_test/parameters.md b/docs/en/user_guides/stress_test/parameters.md index 0ee272bec..736840d55 100644 --- a/docs/en/user_guides/stress_test/parameters.md +++ b/docs/en/user_guides/stress_test/parameters.md @@ -36,7 +36,7 @@ Execute `evalscope perf --help` to get a full parameter description. | `--stream` | `bool` | Whether to use SSE stream output
Must be enabled to measure TTFT (Time to First Token) metric | `True` | | `--sleep-interval` | `int` | Sleep time between each performance test (seconds)
Helps avoid overloading the server | `5` | | `--open-loop` | `bool` | Enable open-loop mode: dispatch requests following a Poisson arrival schedule without semaphore backpressure.
Requests are fired at the rate set by `--rate` regardless of whether the server has finished processing previous requests.
• `--rate` becomes the sweep variable (accepts multiple values), replacing `--parallel` to drive multi-run iterations
• `--number` must have the same length as `--rate`; each pair `(rate, number)` corresponds to one independent run
• `--parallel` is ignored in this mode (internally set to -1 / INF)
See [Usage Example](./examples.md#open-loop-mode) | `False` | -| `--warmup-num` | `float` | Number or ratio of warmup requests:
• `0`: disabled (default)
• `>= 1`: absolute count, e.g. `--warmup-num 10` sends 10 warmup requests
• `0 < value < 1`: ratio mode, e.g. `--warmup-num 0.1` = 10% of `--number`
Warmup requests are sent with the same concurrency/rate as the benchmark but **excluded from performance metrics**
Useful for eliminating cold-start effects (KV-cache filling, JIT compilation, etc.)
See [Usage Example](./examples.md#warmup-benchmarking) | `0` | +| `--warmup-num` | `float` | Number or ratio of warmup requests:
• `0`: disabled (default)
• `>= 1`: absolute count, e.g. `--warmup-num 10` sends 10 warmup requests
• `0 < value < 1`: ratio mode, e.g. `--warmup-num 0.1` = 10% of `--number`
Warmup requests are sent with the same concurrency/rate as the benchmark but **excluded from performance metrics**
Useful for eliminating cold-start effects (KV-cache filling, JIT compilation, etc.)
See [Usage Example](./examples.md#warmup) | `0` | | `--duration` | `float` | Wall-clock budget for one benchmark run (seconds)
Soft-exit semantics: once the deadline elapses **no new requests are dispatched**, but **already in-flight requests are allowed to finish** before exit
In multi-turn mode "in-flight" means **already-claimed traces run every remaining turn** (trace-level soft exit, aligned with upstream trie)
When combined with `--number`, **whichever cap is hit first** ends the run | `None` | ```{tip} @@ -62,36 +62,34 @@ Execute `evalscope perf --help` to get a full parameter description. | `--sla-fixed-parallel` | `int` | Fixed parallel workers used when `--sla-variable=rate`; defaults to `--sla-upper-bound` for backward compatibility | `None` | | `--sla-num-runs` | `int` | Number of runs per concurrency level (average taken) | `3` | | `--sla-number-multiplier` | `float` | Multiplier of total requests relative to the tuned variable (concurrency or rate), i.e. `number = round(variable × N)`; defaults to `2` when not set | `None` | + ```{seealso} For details on using the SLA auto-tuning feature, see the [Auto-tuning Guide](./sla_auto_tune.md). ``` -## Prompt Settings - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `--max-prompt-length` | `int` | Maximum input prompt length
Prompts exceeding this length will be discarded | `131072` | -| `--min-prompt-length` | `int` | Minimum input prompt length
Prompts shorter than this will be discarded | `0` | -| `--prefix-length` | `int` | Length of the prompt prefix
Only effective for `random` dataset | `0` | -| `--prompt` | `str` | Specify request prompt
String or local file (specify via `@/path/to/file`)
Higher priority than `dataset`
Example: `@./prompt.txt` | - | -| `--query-template` | `str` | Specify query template
JSON string or local file (specify via `@/path/to/file`)
Example: `@./query_template.json` | - | -| `--apply-chat-template` | `bool` | Whether to apply chat template | `None` (automatically determined based on URL suffix) | -| `--image-width` | `int` | Image width for random VL dataset | `224` | -| `--image-height` | `int` | Image height for random VL dataset | `224` | -| `--image-format` | `str` | Image format for random VL dataset | `RGB` | -| `--image-num` | `int` | Number of images for random VL dataset | `1` | -| `--image-patch-size` | `int` | Patch size for the image
Only used for local image token calculation | `28` | - ## Dataset Configuration | Parameter | Type | Description | Default | |-----------|------|-------------|--------| -| `--dataset` | `str` | Dataset mode, see table below for details | - | +| `--dataset` | `str` | Dataset mode, see [Dataset Modes](#dataset-modes) below | - | | `--dataset-path` | `str` | Dataset file or directory path
Points to a file: read directly; points to a directory: looks for the corresponding data file inside (for offline use with pre-downloaded dataset cache) | - | | `--data-source` | `str` | Data source for dataset loading: `modelscope`, `huggingface`, or `local`
Defaults to `modelscope` when not specified; automatically treated as `local` when `--dataset-path` is a local directory | `modelscope` | -| `--dataset-args` | `str` | Per-dataset arguments (JSON string), validated against the schema of the selected `--dataset` (unknown keys raise an error). Carries the fixed-length input args `target_input_len` / `input_len_mode` for real-text datasets, and supersedes the deprecated `--multi-turn-args`. See the "dataset-args: Per-dataset Arguments" section below | - | +| `--dataset-args` | `str` | Per-dataset arguments (JSON string), validated against the schema of the selected `--dataset` (unknown keys raise an error) | - | -### Dataset Mode Description +The keys carried by `--dataset-args` are documented in the sections where they apply: + +| Key | Purpose | Section | +|-----|---------|---------| +| `target_input_len` / `input_len_mode` | Truncate real data to a fixed input length | [Length Control](#length-control) | +| `prefix_file` / `prefix_role` | Long-context prefix injection for very long fixed-length inputs | [Long-context Prefix Injection](#long-context-prefix-injection) | +| `speed` / `model_override` / `model_mapping` / `match_output_length` | Replay behaviour for production traffic | [Production Traffic Replay](#production-traffic-replay) | +| Token-length arguments | Multi-turn datasets | [Multi-turn Conversation](#multi-turn-conversation) | + +```{note} +`--multi-turn-args` is deprecated; use `--dataset-args` instead (the key names are unchanged). The old flag still works and is automatically merged into `--dataset-args` (on a key conflict, `--dataset-args` takes precedence). +``` + +### Dataset Modes **Text / Chat** @@ -100,7 +98,7 @@ For details on using the SLA auto-tuning feature, see the [Auto-tuning Guide](./ | `openqa` | Automatically downloads [OpenQA](https://www.modelscope.cn/datasets/AI-ModelScope/HC3-Chinese/summary) from ModelScope
Prompts are relatively short (usually <100 tokens)
Uses `question` field from jsonl file when `dataset_path` is specified | ✓ | | `longalpaca` | Automatically downloads [LongAlpaca-12k](https://www.modelscope.cn/datasets/AI-ModelScope/LongAlpaca-12k/dataPeview) from ModelScope
Prompts are much longer (generally >6000 tokens)
Uses `instruction` field from jsonl file when `dataset_path` is specified | ✓ | | `line_by_line` | Each line in txt file is used as a separate prompt
**Requires `dataset_path`** | ✓ (Required) | -| `random` | Randomly generates prompts based on `prefix-length`, `max-prompt-length`, and `min-prompt-length`
**Requires `tokenizer-path`**
[Usage example](./examples.md#using-the-random-dataset) | ✗ | +| `random` | Randomly generates prompts based on `prefix-length`, `max-prompt-length`, and `min-prompt-length`
**Requires `tokenizer-path`**
[Usage example](./examples.md#random-dataset) | ✗ | | `custom` | Custom dataset parser
See [Custom Dataset Guide](custom.md#custom-dataset) | ✓ | **Multimodal** @@ -109,7 +107,7 @@ For details on using the SLA auto-tuning feature, see the [Auto-tuning Guide](./ |------|-------------|----------------------| | `flickr8k` | Automatically downloads [Flick8k](https://www.modelscope.cn/datasets/clip-benchmark/wds_flickr8k/dataPeview) from ModelScope
Builds image-text inputs; large dataset suitable for evaluating multimodal models
Supports `--dataset-path` pointing to a local dataset directory (offline) | ✓ (directory) | | `kontext_bench` | Automatically downloads [Kontext-Bench](https://modelscope.cn/datasets/black-forest-labs/kontext-bench/dataPeview) from ModelScope
Builds image-text inputs; approximately 1,000 samples, suitable for quick evaluation of multimodal models
Supports `--dataset-path` pointing to a local dataset directory (offline) | ✓ (directory) | -| `random_vl` | Randomly generates both image and text inputs
Based on `random`, with additional image-related parameters
[Usage example](./examples.md#using-the-random-multimodal-dataset) | ✗ | +| `random_vl` | Randomly generates both image and text inputs
Based on `random`, with additional image-related parameters
[Usage example](./examples.md#random-multimodal-dataset) | ✗ | **Embedding** @@ -129,7 +127,7 @@ For details on using the SLA auto-tuning feature, see the [Auto-tuning Guide](./ **Multi-turn Conversation** -Must be used with `--multi-turn`. See the [Multi-turn Benchmark Guide](./multi_turn.md) for details. +Must be used with `--multi-turn`; see [Multi-turn Conversation](#multi-turn-conversation) for the parameters and the [Multi-turn Benchmark Guide](./multi_turn.md) for details. | Mode | Description | Supports dataset-path | |------|-------------|----------------------| @@ -140,19 +138,29 @@ Must be used with `--multi-turn`. See the [Multi-turn Benchmark Guide](./multi_t **Production Traffic Replay** -Must be used with `--open-loop`; replays recorded requests verbatim following their **original arrival timing**. See [Usage example](./examples.md#production-traffic-replay-workload_trace). +Must be used with `--open-loop`; see [Production Traffic Replay](#production-traffic-replay) for the trace file format and replay arguments. | Mode | Description | Supports dataset-path | |------|-------------|----------------------| | `workload_trace` | Replays a recorded production-traffic JSONL verbatim — original timestamps, request bodies, and headers — for benchmarking against real-world load (bursty arrivals, heterogeneous requests, multi-model routing)
Each request carries its own `model`, preserving multi-model routing
**Requires `--open-loop` and `--dataset-path`**; `--model`/`--number` are optional (the trace carries its own model and count) | ✓ (Required) | -### dataset-args: Per-dataset Arguments +## Input Construction + +Controls the content and length of the input sent to the model. Below, `--xxx` entries are command-line flags while bare keys are passed through the `--dataset-args` JSON. + +### Length Control -`--dataset-args` passes per-dataset arguments as a JSON string (a mistyped key raises an error, making mistakes easy to spot). Two common use cases: +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `--max-prompt-length` | `int` | Maximum input prompt length
Prompts exceeding this length will be discarded | `131072` | +| `--min-prompt-length` | `int` | Minimum input prompt length
Prompts shorter than this will be discarded | `0` | -**Case 1: Truncate real data to a fixed input length** +To benchmark a **fixed input length** with real data (instead of `random`), use the following `--dataset-args` keys. Supported on `openqa`, `longalpaca`, `line_by_line` (plain-text lines only), and ShareGPT (`share_gpt_zh` / `share_gpt_en`). **Requires `--tokenizer-path`**. -Use it when you want to benchmark a **fixed input length** with real data (instead of `random`). Supported on `openqa`, `longalpaca`, `line_by_line`, and single-turn ShareGPT (`share_gpt_zh` / `share_gpt_en`). **Requires `--tokenizer-path`**. +| Key | Description | Default | +|-----|-------------|--------| +| `target_input_len` | Target input length in tokens. When set, every prompt is truncated to this length | disabled | +| `input_len_mode` | What to do with prompts **shorter than** the target: `cap` (keep as-is; that prompt may be shorter than the target); `drop` (discard it, so every emitted prompt is exactly the target). `drop` cannot be combined with `prefix_file` (see [Long-context Prefix Injection](#long-context-prefix-injection)) | `cap` | ```bash # Truncate every input to 2048 tokens @@ -162,10 +170,7 @@ evalscope perf \ --dataset-args '{"target_input_len": 2048}' ``` -| Key | Description | Default | -|-----|-------------|--------| -| `target_input_len` | Target input length in tokens. When set, every prompt is truncated to this length | disabled | -| `input_len_mode` | What to do with prompts **shorter than** the target: `cap` (keep as-is; that prompt may be shorter than the target); `drop` (discard it, so every emitted prompt is exactly the target) | `cap` | +Lengths are counted as bare content tokens, without chat-template overhead. Multi-turn datasets like ShareGPT fit/filter only the **last user turn** by default (same as single-turn); the whole-conversation budget (sum of all message contents, over-long dropped, short padded by the prefix) applies only when a `prefix_file` is configured (see [Long-context Prefix Injection](#long-context-prefix-injection)). JSON lines in `line_by_line` (messages array / full request body) do not go through length control, so combining them with these keys raises an error. How it differs from `--max/min-prompt-length`: - `--max/min-prompt-length` only **filters** and never changes content — samples outside the range are dropped, so you get real samples of varying lengths; @@ -173,19 +178,81 @@ How it differs from `--max/min-prompt-length`: > To get "every prompt exactly N tokens", you must use `target_input_len`; setting `--min-prompt-length` equal to `--max-prompt-length` cannot achieve it (real data almost never has prompts of exactly N tokens, so they get filtered out). The `random` dataset is the exception — it is generated on the fly, so min=max already yields a fixed length and this arg is not needed. -**Case 2: Length arguments for multi-turn datasets** +### Long-context Prefix Injection -Token-length arguments for multi-turn datasets such as `swe_smith` are also passed via `--dataset-args`; see the [Multi-turn Benchmark Guide](./multi_turn.md). +Real instruction datasets are mostly 4K-8K tokens, so setting `target_input_len` to 128K filters everything out in `drop` mode and leaves varying lengths in `cap` mode. `prefix_file` lets you point at a long text (e.g. a book or document corpus); the framework slices it to exactly `target_input_len − prompt length` tokens and prepends it to each short prompt, so **every request's total input hits the target length** while keeping the low-entropy character of real human language (ideal for measuring prefix-cache hit rates and MTP acceptance rates). Supported on the same datasets as [Length Control](#length-control). -**Case 3: Replay real production traffic** +| Key | Description | Default | +|-----|-------------|--------| +| `prefix_file` | Path to the long prefix text file (UTF-8 plain text). **Requires `target_input_len`** and cannot be combined with `input_len_mode="drop"` | disabled | +| `prefix_role` | Injection role for the prefix: `system` (injected as a leading system message, matching real RAG traffic; inference engines usually have dedicated prefix-cache handling for system prompts); `user` (prepended directly to the user message content) | `system` | -Use `workload_trace` to replay recorded production traffic verbatim following its **original arrival timing**, closely matching real-world load. **Requires `--open-loop`**; no `--rate` is needed (arrival times come from the trace timestamps). See the full example at [Production Traffic Replay (workload_trace)](./examples.md#production-traffic-replay-workload_trace). +```bash +# Align every request to exactly 131072 tokens using a long text prefix injected as the system role +evalscope perf \ + --model qwen2.5 --url http://127.0.0.1:8000/v1/chat/completions \ + --dataset openqa --tokenizer-path /path/to/tokenizer \ + --dataset-args '{"target_input_len": 131072, "prefix_file": "/path/to/long_text.txt", "prefix_role": "system"}' +``` + +Behavior notes: +- **Budget split**: the prompt is kept as-is (over-length prompts are truncated per `input_len_mode`); the prefix is sliced to exactly `target_input_len − tokens of all message contents`, so the total equals the target. Multi-turn history counts towards the budget (see the length convention in [Length Control](#length-control)). +- **Incompatible with `drop`**: `drop` only keeps prompts that already fill `target_input_len`, leaving a zero prefix budget, so the injection could never take effect — the combination is rejected at config time. Use `cap` plus prefix filling for fixed lengths. +- **Short prefix**: when the prefix file has fewer tokens than the remaining budget, it is repeated (tiled) to cover it and then truncated precisely, with a warning. +- **Fallback**: when `apply_chat_template` is off (e.g. the `/v1/completions` endpoint), a system message cannot be injected, so the prefix falls back to plain-text concatenation with a warning. +- **Join boundary**: with `prefix_role="user"` and in the plain-text fallback the prefix sits directly next to the prompt. The prefix and prompt are counted independently, so when the tokenizer merges or splits characters across the join the measured total can differ from the target by about ±1 token. `prefix_role="system"` (chat-template mode) is separated by message markers and is always exact. +- **Cache friendly**: all requests share the same prefix head (lengths differ slightly per prompt), which naturally suits prefix-cache hit testing. + +```{note} +The `prefix_file` in this section injects a **real text** prefix for real datasets, whereas `--prefix-length` injects a **random token** prefix and only applies to the `random` dataset (see [Random Data Generation](#random-data-generation) below). They serve different purposes and should not be confused. +``` + +### Random Data Generation + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `--prefix-length` | `int` | Length of the random token prefix
Only effective for the `random` dataset; all requests share the same prefix, which can be used to induce prefix-cache hits | `0` | +| `--image-width` | `int` | Image width for random VL dataset | `224` | +| `--image-height` | `int` | Image height for random VL dataset | `224` | +| `--image-format` | `str` | Image format for random VL dataset | `RGB` | +| `--image-num` | `int` | Number of images for random VL dataset | `1` | +| `--image-patch-size` | `int` | Patch size for the image
Only used for local image token calculation | `28` | + +The length of `random` prompts is set by `--min-prompt-length` / `--max-prompt-length` (equal values yield a fixed length); `target_input_len` is not needed. + +### Prompt and Template + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `--prompt` | `str` | Specify request prompt
String or local file (specify via `@/path/to/file`)
Higher priority than `dataset`
Example: `@./prompt.txt` | - | +| `--query-template` | `str` | Specify query template
JSON string or local file (specify via `@/path/to/file`)
Example: `@./query_template.json` | - | +| `--apply-chat-template` | `bool` | Whether to apply chat template | `None` (automatically determined based on URL suffix) | +| `--tokenize-prompt` | `bool` | Tokenize the prompt client-side into a token-ID list and send it directly via `/v1/completions`, bypassing server-side re-tokenization | `False` | + +## Multi-turn Conversation + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `--multi-turn` | `bool` | Enable multi-turn conversation benchmark mode; `--number` is the total number of turns to send and `--parallel` is the number of concurrent turn-level requests | `False` | +| `--min-turns` | `int` | Minimum number of user turns per conversation; used by `random_multi_turn` and `swe_smith` | `1` | +| `--max-turns` | `int` | Maximum number of user turns per conversation; required for `random_multi_turn`; optional for ShareGPT / `custom_multi_turn` (truncates long conversations); for `swe_smith` it's the upper bound for per-conversation turn sampling, falling back to `--min-turns` when unset | `None` | +| `--num-workers` | `int` | Worker processes for CPU-bound dataset/request generation.
`0` = auto-detect from CPU affinity; `1` = serial (no multiprocessing); `>1` = explicit worker count.
Used by `random` (long-prompt parallel generation) and `swe_smith` (live construction). Supersedes the deprecated `multi_turn_args.num_workers`. | `0` | + +Token-length arguments for multi-turn datasets such as `swe_smith` are passed via `--dataset-args`. + +```{seealso} +Available multi-turn datasets are listed in [Dataset Modes](#dataset-modes); for full usage see the [Multi-turn Benchmark Guide](./multi_turn.md). +``` + +## Production Traffic Replay + +Use `--dataset workload_trace` to replay recorded production traffic verbatim following its **original arrival timing**, closely matching real-world load. **Requires `--open-loop`**; no `--rate` is needed (arrival times come from the trace timestamps). See the full example at [Production Traffic Replay](./examples.md#production-traffic-replay). The trace file is JSONL, one request record per line: -```jsonl +```json {"body": {"model": "qwen-plus", "messages": [{"role": "user", "content": "hi"}]}, "timestamp": 1700000000.0} -{"body": {"model": "qwen-max", "messages": [...]}, "timestamp": 1700000001.5, "headers": {"X-Tag": "exp"}, "request_id": "req-42", "completion_tokens": 256} +{"body": {"model": "qwen-max", "messages": [{"role": "user", "content": "hello"}]}, "timestamp": 1700000001.5, "headers": {"X-Tag": "exp"}, "request_id": "req-42", "completion_tokens": 256} ``` | Field | Required | Description | @@ -196,6 +263,8 @@ The trace file is JSONL, one request record per line: | `request_id` | | Propagated to results for correlation with the original request | | `completion_tokens` | | Used together with `match_output_length` | +Replay behaviour is tuned via `--dataset-args`: + | Key | Type | Description | Default | |-----|------|-------------|--------| | `speed` | float | Replay speed multiplier (2.0 = 2× faster, 0.5 = 2× slower) | `1.0` | @@ -207,18 +276,14 @@ The trace file is JSONL, one request record per line: `--model` **does not rewrite** the trace body for `workload_trace` — each request keeps its own `model`, preserving multi-model routing. To rewrite models, use `model_override` / `model_mapping`. ``` -```{note} -`--multi-turn-args` is deprecated; use `--dataset-args` instead (the key names are unchanged). The old flag still works and is automatically merged into `--dataset-args` (on a key conflict, `--dataset-args` takes precedence). -``` - -## Model Settings +## Model and Generation | Parameter | Type | Description | Default | |-----------|------|-------------|---------| | `--tokenizer-path` | `str` | Tokenizer weights path
Used to calculate the number of tokens in input and output
Usually located in the same directory as model weights | `None` | | `--frequency-penalty` | `float` | frequency_penalty value | - | | `--logprobs` | `bool` | Whether to return logarithmic probabilities | - | -| `--max-tokens` | `int` | Maximum number of tokens that can be generated | - | +| `--max-tokens` | `int` or `int int` | Maximum number of tokens that can be generated
• A single integer: fixed value, e.g. `--max-tokens 2048`
• Two integers: `min max`, sampled uniformly at random per request, e.g. `--max-tokens 512 2048` | `2048` | | `--min-tokens` | `int` | Minimum number of tokens to generate
Note: Not all model services support this parameter
For `vLLM>=0.8.1`, you need to additionally set
`--extra-args '{"ignore_eos": true}'` | - | | `--n-choices` | `int` | Number of completion choices to generate | - | | `--seed` | `int` | Random seed | `None` | @@ -228,9 +293,8 @@ The trace file is JSONL, one request record per line: | `--top-p` | `float` | Top-p sampling | - | | `--top-k` | `int` | Top-k sampling | - | | `--extra-args` | `str` | Additional parameters to be passed in the request body
JSON string format
Example: `'{"ignore_eos": true}'` | - | -| `--tokenize-prompt` | `bool` | Tokenize the prompt client-side into a token-ID list and send it directly via `/v1/completions`, bypassing server-side re-tokenization | `False` | -## Data Storage +## Output | Parameter | Type | Description | Default | |-----------|------|-------------|---------| @@ -241,19 +305,6 @@ The trace file is JSONL, one request record per line: | `--outputs-dir` | `str` | Output file path | `./outputs` | | `--no-timestamp` | `bool` | Exclude timestamp from output directory name | `False` | -## Multi-turn Settings - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `--multi-turn` | `bool` | Enable multi-turn conversation benchmark mode; `--number` is the total number of turns to send and `--parallel` is the number of concurrent turn-level requests | `False` | -| `--min-turns` | `int` | Minimum number of user turns per conversation; used by `random_multi_turn` and `swe_smith` | `1` | -| `--max-turns` | `int` | Maximum number of user turns per conversation; required for `random_multi_turn`; optional for ShareGPT / `custom_multi_turn` (truncates long conversations); for `swe_smith` it's the upper bound for per-conversation turn sampling, falling back to `--min-turns` when unset | `None` | -| `--num-workers` | `int` | Worker processes for CPU-bound dataset/request generation.
`0` = auto-detect from CPU affinity; `1` = serial (no multiprocessing); `>1` = explicit worker count.
Used by `random` (long-prompt parallel generation) and `swe_smith` (live construction). Supersedes the deprecated `multi_turn_args.num_workers`. | `0` | - -```{seealso} -For details on using the multi-turn benchmark feature, see the [Multi-turn Benchmark Guide](./multi_turn.md). -``` - ## Other Parameters | Parameter | Type | Description | Default | diff --git a/docs/en/user_guides/stress_test/quick_start.md b/docs/en/user_guides/stress_test/quick_start.md index c9236bac5..85e69c21d 100644 --- a/docs/en/user_guides/stress_test/quick_start.md +++ b/docs/en/user_guides/stress_test/quick_start.md @@ -79,7 +79,7 @@ results = run_perf_benchmark(task_cfg) - `url`: Request URL address - `model`: Name of the model used - `api`: API service used, default is `openai` -- `dataset`: Dataset name, here it's `random`, indicating randomly generated dataset, for specific usage instructions [refer to](./examples.md#using-the-random-dataset); for more available (multimodal) datasets, please refer to [Dataset Configuration](./parameters.md#dataset-configuration) +- `dataset`: Dataset name, here it's `random`, indicating randomly generated dataset, for specific usage instructions [refer to](./examples.md#random-dataset); for more available (multimodal) datasets, please refer to [Dataset Configuration](./parameters.md#dataset-configuration) - `tokenizer-path`: Model's tokenizer path, used to calculate the number of tokens (necessary for random datasets) - `extra-args`: Additional parameters in the request, passed as a JSON format string, e.g., `{"ignore_eos": true}` indicates ignoring the end token @@ -204,7 +204,7 @@ The following metrics are only shown in speculative decoding scenarios. #### Per-Trace and Workload Throughput Metrics -In multi-turn mode, two additional tables are output: **Per-Trace Metrics** (trace-level distribution metrics) and **Workload Throughput** (global token throughput rates across time windows). For details, see [Multi-Turn Stress Testing — Multi-Turn Output Metrics](./multi_turn.md#multi-turn-output-metrics). +In multi-turn mode, two additional tables are output: **Per-Trace Metrics** (trace-level distribution metrics) and **Workload Throughput** (global token throughput rates across time windows). For details, see [Multi-Turn Stress Testing — Output Metrics](./multi_turn.md#output-metrics). ## Visualizing Test Results diff --git a/docs/zh/advanced_guides/custom_dataset/clip.md b/docs/zh/advanced_guides/custom_dataset/clip.md index 3f55128e4..9815f63be 100644 --- a/docs/zh/advanced_guides/custom_dataset/clip.md +++ b/docs/zh/advanced_guides/custom_dataset/clip.md @@ -44,7 +44,7 @@ task_cfg = { } ``` ```{seealso} -[全部参数说明](../../user_guides/backend/rageval_backend/clip_benchmark.md#配置评测参数) +[完整参数参考](../../user_guides/backend/rageval_backend/clip_benchmark.md#完整参数参考) ``` 其中： - `dataset_name`: 数据集名称，必须指定为`custom`。 diff --git a/docs/zh/advanced_guides/custom_dataset/embedding.md b/docs/zh/advanced_guides/custom_dataset/embedding.md index cc8b9dce9..c29e35cdc 100644 --- a/docs/zh/advanced_guides/custom_dataset/embedding.md +++ b/docs/zh/advanced_guides/custom_dataset/embedding.md @@ -148,7 +148,7 @@ python run_eval.py | `eval_splits` | List[str] | `["test"]` | 评测分割列表 | ```{note} -其他评测参数（如 `models`、`limits`、`overwrite_results` 等）与默认配置一致，详见 [MTEB 评测参数说明](../../user_guides/backend/rageval_backend/mteb.md#参数说明)。 +其他评测参数（如 `models`、`limits`、`overwrite_results` 等）与默认配置一致，详见 [MTEB 评测参数说明](../../user_guides/backend/rageval_backend/mteb.md#完整参数参考)。 ``` ## 常见问题 diff --git a/docs/zh/get_started/parameters.md b/docs/zh/get_started/parameters.md index b71409b4c..af58d45c0 100644 --- a/docs/zh/get_started/parameters.md +++ b/docs/zh/get_started/parameters.md @@ -111,7 +111,7 @@ | `aggregation` | `str` | 评测结果聚合方式，默认`mean`。可选：`mean_and_pass_at_k`、`mean_and_vote_at_k`、`mean_and_pass_hat_k`（均需设置`repeats=k`）。
• `pass_at_k`：同一样例生成k次至少一次通过的概率（如`humaneval`设`repeats=5`）
• `vote_at_k`：对同一样例k次结果投票后计分
• `pass_hat_k`：同一样例k次全部通过的概率（如`tau2_bench`设`repeats=3`） | | `filters` | `dict` | 输出过滤器
• `remove_until`: 过滤指定字符串之前的内容
• `extract`: 提取正则匹配的内容 | | `force_redownload` | `bool` | 是否强制重新下载数据集 | -| `extra_params` | `dict` | 数据集相关的**额外参数**，具体参考[各数据集说明](./supported_dataset/index.md)，指定`{:}`即可, `value`的类型(`type`)和选择范围(`choices`)根据具体参数而定。SWE-bench agentic 等基准的扩展参数请参见 [Agent 评测](../user_guides/agent/native.md#swe-bench-agentic-基准) | +| `extra_params` | `dict` | 数据集相关的**额外参数**，具体参考[各数据集说明](./supported_dataset/index.md)，指定`{:}`即可, `value`的类型(`type`)和选择范围(`choices`)根据具体参数而定。SWE-bench agentic 等基准的扩展参数请参见 [Agent 评测](../user_guides/agent/native.md#用例swe-bench-agentic) | | `sandbox_config` | `dict` | Sandbox配置（详见下方Sandbox参数） | **sandbox_config 配置项：** diff --git a/docs/zh/third_party/deepsearchqa.md b/docs/zh/third_party/deepsearchqa.md index 70ab615e9..58e664636 100644 --- a/docs/zh/third_party/deepsearchqa.md +++ b/docs/zh/third_party/deepsearchqa.md @@ -125,4 +125,4 @@ evalscope eval \ - [论文](https://storage.googleapis.com/deepmind-media/DeepSearchQA/DeepSearchQA_benchmark_paper.pdf) - [ModelScope 数据集](https://modelscope.cn/datasets/google/deepsearchqa/summary) -- [Agent MCP 配置指南](../user_guides/agent/native.md#mcp-server-tools) +- [Agent MCP 配置指南](../user_guides/agent/native.md#mcp-工具接入) diff --git a/docs/zh/user_guides/stress_test/examples.md b/docs/zh/user_guides/stress_test/examples.md index 067d0bdec..372e78ca0 100644 --- a/docs/zh/user_guides/stress_test/examples.md +++ b/docs/zh/user_guides/stress_test/examples.md @@ -1,152 +1,77 @@ # 使用示例 -## 使用本地模型推理 +本文按「压测对象 → 输入数据 → 请求参数 → 负载模式 → 结果观测」的顺序给出可直接复制运行的命令。完整参数含义见[参数说明](./parameters.md)，多轮对话场景见[多轮对话压测](./multi_turn.md)。 -本项目支持本地transformers进行推理和vllm推理（需先安装vllm）， `--model`可以填入modelscope模型名称，例如`Qwen/Qwen2.5-0.5B-Instruct`；也可以直接指定模型权重路径，例如`/path/to/model_weights`，无需指定`--url`参数。 +## 本地模型推理 -**1. 使用transformers进行推理** +支持本地 transformers 推理和 vLLM 推理（需先安装 vllm），无需指定 `--url`。`--model` 可填 ModelScope 模型名称（如 `Qwen/Qwen2.5-0.5B-Instruct`），也可直接指定模型权重路径（如 `/path/to/model_weights`）。 + +**transformers 推理**：指定 `--api local`。 -指定`--api local`： ```bash evalscope perf \ - --model 'Qwen/Qwen2.5-0.5B-Instruct' \ - --attn-implementation flash_attention_2 \ # 可不填，或选[flash_attention_2|eager|sdpa] - --number 20 \ - --parallel 2 \ - --api local \ - --dataset openqa + --model 'Qwen/Qwen2.5-0.5B-Instruct' \ + --number 20 \ + --parallel 2 \ + --api local \ + --dataset openqa ``` -**2. 使用vllm进行推理** +可选追加 `--attn-implementation`，取值 `flash_attention_2`、`eager` 或 `sdpa`。 -指定`--api local_vllm`： -```bash -evalscope perf \ - --model 'Qwen/Qwen2.5-0.5B-Instruct' \ - --number 20 \ - --parallel 2 \ - --api local_vllm \ - --dataset openqa -``` +**vLLM 推理**：指定 `--api local_vllm`。 -## 使用`prompt` -```bash -evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --number 20 \ - --api openai \ - --temperature 0.9 \ - --max-tokens 1024 \ - --prompt '写一个科幻小说，请开始你的表演' -``` -也可以使用本地文件作为prompt： ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --number 20 \ - --api openai \ - --temperature 0.9 \ - --max-tokens 1024 \ - --prompt @prompt.txt + --model 'Qwen/Qwen2.5-0.5B-Instruct' \ + --number 20 \ + --parallel 2 \ + --api local_vllm \ + --dataset openqa ``` -## 复杂请求 -使用`stop`，`stream`，`temperature`等： +## 输入数据构造 -```bash -evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --read-timeout 120 \ - --connect-timeout 120 \ - --number 20 \ - --max-prompt-length 128000 \ - --min-prompt-length 128 \ - --api openai \ - --temperature 0.7 \ - --max-tokens 1024 \ - --stop '<|im_end|>' \ - --dataset openqa \ - --stream -``` - -## 使用`query-template` - -您可以在`query-template`中设置请求参数： +### 固定 prompt + +用 `--prompt` 指定单条固定 prompt，所有请求发送相同内容，无需数据集。 ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --read-timeout 120 \ - --connect-timeout 120 \ - --number 20 \ - --max-prompt-length 128000 \ - --min-prompt-length 128 \ - --api openai \ - --query-template '{"model": "%m", "messages": [{"role": "user","content": "%p"}], "stream": true, "skip_special_tokens": false, "stop": ["<|im_end|>"], "temperature": 0.7, "max_tokens": 1024}' \ - --dataset openqa -``` -其中`%m`和`%p`会被替换为模型名称和prompt。 - -您也可以使用本地`query-template.json`文件： + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --number 20 \ + --api openai \ + --temperature 0.9 \ + --max-tokens 1024 \ + --prompt '写一个科幻小说，请开始你的表演' +``` -```{code-block} json -:caption: template.json +也可以用 `@` 前缀从本地文件读取： -{ - "model":"%m", - "messages":[ - { - "role":"user", - "content":"%p" - } - ], - "stream":true, - "skip_special_tokens":false, - "stop":[ - "<|im_end|>" - ], - "temperature":0.7, - "max_tokens":1024 -} -``` ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 2 \ - --model 'qwen2.5' \ - --log-every-n-query 10 \ - --read-timeout 120 \ - --connect-timeout 120 \ - --number 20 \ - --max-prompt-length 128000 \ - --min-prompt-length 128 \ - --api openai \ - --query-template @template.json \ - --dataset openqa + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --number 20 \ + --api openai \ + --temperature 0.9 \ + --max-tokens 1024 \ + --prompt @prompt.txt ``` -## 使用random数据集 +### 随机数据集 -根据`prefix-length`，`max-prompt-length`和`min-prompt-length`随机生成prompt，必需指定`tokenizer-path`。生成prompt的token数量在`prefix_length + min-prompt-length`和`prefix_length + max-prompt-length`之间均匀分布，在一次测试中所有请求prefix部分相同。 +根据 `prefix-length`、`max-prompt-length` 和 `min-prompt-length` 随机生成 prompt，必需指定 `tokenizer-path`。生成 prompt 的 token 数量在 `prefix_length + min-prompt-length` 和 `prefix_length + max-prompt-length` 之间均匀分布，在一次测试中所有请求 prefix 部分相同。 ```{note} -由于chat_template以及tokenize算法的影响，生成的prompt的token数量可能有些误差，不是精确的指定token数量。 +由于 chat_template 以及 tokenize 算法的影响，生成的 prompt 的 token 数量可能有些误差，不是精确的指定 token 数量。 ``` -执行以下命令即可： - ```bash evalscope perf \ --parallel 20 \ @@ -170,8 +95,9 @@ evalscope perf \ 服务端将收到恰好 `prefix_length + inner_seq_length` 个 token，落在 `[min-prompt-length, max-prompt-length]` 范围内。适用于 vLLM、SGLang、LMDeploy 等支持接收 token ID 的推理框架；不支持 `random_vl` 数据集。 ``` -## 使用random图文数据集 -使用`random_vl`数据集，随机生成图像和文本输入，在`random`基础上增加了图像相关参数（`image-width`，`image-height`，`image-format`，`image-num`）。 +### 随机图文数据集 + +使用 `random_vl` 数据集，随机生成图像和文本输入，在 `random` 基础上增加了图像相关参数（`image-width`、`image-height`、`image-format`、`image-num`）。 ```bash evalscope perf \ @@ -194,48 +120,187 @@ evalscope perf \ --debug ``` -## Embedding模型压测 +### 长上下文前缀注入 + +想用**真实语料**压测 128K/256K 级长上下文时，`random` 数据集虽然能定长，但生成的是高熵无意义 token，无法反映 Prefix-Cache 命中率、MTP 接受率等真实特征；而真实指令集大多只有 4K-8K token，直接把 `target_input_len` 设成 128K 会导致数据被筛空或长短不一。 + +`prefix_file` 解决这个问题：指定一份长文本语料（书籍、文档、代码库等），框架按 token 预算把它精确切成 `target_input_len − prompt 长度` 的前缀，与短 prompt 拼接，使**每条请求的输入恰好等于目标长度**，同时保持真实人类语言的低熵特征。参数说明见[长上下文前缀注入](./parameters.md#长上下文前缀注入)。 + +**前缀注入到 system 角色**（推荐，贴近真实 RAG 流量）： + +```bash +evalscope perf \ + --parallel 4 \ + --model Qwen2.5-0.5B-Instruct \ + --url http://127.0.0.1:8801/v1/chat/completions \ + --api openai \ + --dataset openqa \ + --tokenizer-path Qwen/Qwen2.5-0.5B-Instruct \ + --dataset-args '{"target_input_len": 8192, "prefix_file": "long_text.txt", "prefix_role": "system"}' \ + --max-tokens 128 \ + --number 20 +``` + +每条请求被构造成 `[{"role": "system", "content": "<长前缀>"}, {"role": "user", "content": "<原始问题>"}]`，服务端上报的 `Input Tokens` 各条完全一致（`avg` = `p50` = `p99` = `max`），便于做单一长度点的性能对照。 + +**前缀拼进 user 消息**：把 `prefix_role` 改为 `user`，前缀直接拼在用户问题前面，产出单条 user 消息。适用于不希望引入 system 角色的场景。 + +```bash +evalscope perf \ + --parallel 2 \ + --model Qwen2.5-0.5B-Instruct \ + --url http://127.0.0.1:8801/v1/chat/completions \ + --api openai \ + --dataset openqa \ + --tokenizer-path Qwen/Qwen2.5-0.5B-Instruct \ + --dataset-args '{"target_input_len": 131072, "prefix_file": "long_text.txt", "prefix_role": "user"}' \ + --number 10 +``` + +```{note} +**实操要点** + +- **前缀语料准备**：任意 UTF-8 纯文本文件即可。语料 token 数不足以填满预算时会自动循环重复（tile）补齐并打 warning——想完全避免重复，准备一份 token 数大于 `target_input_len` 的语料。 +- **别超过服务端上限**：`target_input_len` 需小于服务端的 `max_model_len`（vLLM 可用 `--max-model-len` 调整），否则请求会被拒绝。注意 chat template 本身还会额外占用十几个 token。 +- **适用数据集**：`openqa`、`longalpaca`、`line_by_line`（仅纯文本行）、ShareGPT（`share_gpt_zh` / `share_gpt_en`）；均需 `--tokenizer-path`。ShareGPT 按整段对话所有消息内容计量预算，总长已超过 `target_input_len` 的对话会被整条丢弃。 +- **不能配 `drop`**：`prefix_file` 与 `input_len_mode="drop"` 互斥（`drop` 只留下已填满目标长度的 prompt，前缀预算恒为 0），配置时会直接报错；定长请用默认的 `cap` 并由前缀补齐。 +- **Prefix-Cache 效果**：所有请求共享同一段前缀开头，重复跑同一配置时可观察到 TTFT 显著下降（推理框架命中前缀缓存）。反之，若要测**无缓存**的冷启动性能，请改用 `random` 数据集（它有 `--dataset-offset` 机制保证各轮 prompt 不同）。 +- **`/v1/completions` 端点**：该端点不应用 chat template，`prefix_role="system"` 会自动降级为纯文本拼接并打 warning；此时前缀与 prompt 直接相接，拼接处可能发生 token 合并/拆分，实测总长与目标相差约 ±1 token（`prefix_role="user"` 同理，详见[参数说明](./parameters.md#长上下文前缀注入)的“拼接边界”）。 +``` + +## 请求参数配置 + +### 生成参数与超时 + +组合使用 `stop`、`stream`、`temperature` 以及读写超时等参数： + +```bash +evalscope perf \ + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --read-timeout 120 \ + --connect-timeout 120 \ + --number 20 \ + --max-prompt-length 128000 \ + --min-prompt-length 128 \ + --api openai \ + --temperature 0.7 \ + --max-tokens 1024 \ + --stop '<|im_end|>' \ + --dataset openqa \ + --stream +``` + +### 自定义请求体 + +用 `--query-template` 直接定义完整的请求体 JSON，其中 `%m` 和 `%p` 会被替换为模型名称和 prompt： + +```bash +evalscope perf \ + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --read-timeout 120 \ + --connect-timeout 120 \ + --number 20 \ + --max-prompt-length 128000 \ + --min-prompt-length 128 \ + --api openai \ + --query-template '{"model": "%m", "messages": [{"role": "user","content": "%p"}], "stream": true, "skip_special_tokens": false, "stop": ["<|im_end|>"], "temperature": 0.7, "max_tokens": 1024}' \ + --dataset openqa +``` + +模板较长时，可写入本地 JSON 文件后用 `@` 前缀引用： -使用`openai_embedding` API模式和`random_embedding`数据集进行压测。使用随机数据集时需指定`tokenizer-path`用于生成指定长度范围的query文本。 +```{code-block} json +:caption: template.json + +{ + "model":"%m", + "messages":[ + { + "role":"user", + "content":"%p" + } + ], + "stream":true, + "skip_special_tokens":false, + "stop":[ + "<|im_end|>" + ], + "temperature":0.7, + "max_tokens":1024 +} +``` ```bash evalscope perf \ - --parallel 2 \ - --number 10 \ - --model 'text-embedding-v4' \ - --url 'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings' \ - --api-key ${DASHSCOPE_API_KEY} \ - --api openai_embedding \ - --dataset random_embedding \ - --min-prompt-length 256 \ - --max-prompt-length 256 \ - --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 2 \ + --model 'qwen2.5' \ + --log-every-n-query 10 \ + --read-timeout 120 \ + --connect-timeout 120 \ + --number 20 \ + --max-prompt-length 128000 \ + --min-prompt-length 128 \ + --api openai \ + --query-template @template.json \ + --dataset openqa ``` -## Rerank模型压测 +## 负载模式 + +### Warmup 预热 + +在正式压测前发送一批预热请求，消除冷启动影响（如 KV-cache 填充、JIT 编译、连接池初始化等），使性能指标更准确。 + +预热请求使用与正式压测相同的并发和速率发送，但**不计入性能指标**（延迟、吞吐、百分位等均排除预热数据）。 + +**绝对数量模式**：`--warmup-num` 取 `>= 1` 的整数，表示预热请求的绝对条数。 + +```bash +evalscope perf \ + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 10 \ + --model 'qwen2.5' \ + --number 100 \ + --warmup-num 10 \ + --api openai \ + --dataset openqa \ + --stream +``` -使用`openai_rerank` API模式和`random_rerank`数据集进行压测。使用随机数据集时需指定`tokenizer-path`用于生成指定长度范围的query文本。 +上述命令会先发送 10 个预热请求，再发送 100 个正式压测请求，指标仅统计后 100 个请求。 -可以通过`extra-args`指定生成数据的参数： -- `num_documents`: 每条query对应的文档数量 -- `document_length_ratio`: 文档长度相对于query长度的倍数 +**比例模式**：`--warmup-num` 取 0~1 之间的浮点数，按 `--number` 的比例计算预热数量，适用于 sweep 模式（多轮 `--number` 不同时自动适配）。 ```bash evalscope perf \ - --parallel 2 \ - --number 10 \ - --model 'qwen3-rerank' \ - --url 'https://dashscope.aliyuncs.com/compatible-api/v1/reranks' \ - --api-key ${DASHSCOPE_API_KEY} \ - --api openai_rerank \ - --dataset random_rerank \ - --min-prompt-length 256 \ - --max-prompt-length 256 \ - --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' \ - --extra-args '{"num_documents": 5, "document_length_ratio": 3}' + --url 'http://127.0.0.1:8000/v1/chat/completions' \ + --parallel 10 \ + --model 'qwen2.5' \ + --number 100 \ + --warmup-num 0.1 \ + --api openai \ + --dataset openqa \ + --stream +``` + +`--warmup-num 0.1` 表示预热数量为 `--number` 的 10%，即 `max(1, int(0.1 * 100)) = 10` 个预热请求。 + +```{note} +**注意事项** + +- 预热请求与正式请求使用相同的数据集和请求参数。 +- 预热期间的进度条会单独显示（`Warmup[...]`），完成后自动切换到正式压测进度条（`Processing[...]`）。 +- 多轮对话模式下，`--warmup-num` 表示预热的对话数量（与 `--number` 语义一致），预热对话内的所有 turn 均不计入指标。 ``` -## Open-loop 开放环路模式 +### Open-loop 开放环路 Open-loop 模式下，请求按泊松到达调度（由 `--rate` 控制）立即发出，不等待服务端返回，从而模拟真实流量中请求到达与服务时间无关的场景。通过一次命令指定多个速率点，可自动扫描吞吐-延迟曲线。 @@ -264,13 +329,13 @@ evalscope perf \ - 本模式与 closed-loop（默认）模式的核心区别：closed-loop 每个 worker 等待响应后再发下一条（背压保护），open-loop 不等待、按调度直接发出（更接近真实流量）。 ``` -## 生产流量回放（workload_trace） +### 生产流量回放 `workload_trace` 数据集把录制的生产流量按**原始到达节奏**逐字回放，贴近真实负载——突发到达、异构请求形状、多模型混合路由，这些是合成数据集（`random`、`openqa` 等）难以复现的。它基于 open-loop 调度，但到达时刻由 trace 里的 `timestamp` 决定，**无需 `--rate`**。 -准备一个 JSONL trace 文件（每行一条请求），字段说明见[参数说明 · 场景三](./parameters.md#dataset-args数据集专属参数)： +准备一个 JSONL trace 文件（每行一条请求），字段说明见[参数说明 · 生产流量回放](./parameters.md#生产流量回放)： -```jsonl +```json {"body": {"model": "qwen-plus", "messages": [{"role": "user", "content": "你好"}]}, "timestamp": 1700000000.0} {"body": {"model": "qwen-max", "messages": [{"role": "user", "content": "写一首诗"}]}, "timestamp": 1700000001.5, "request_id": "req-42"} ``` @@ -317,60 +382,55 @@ evalscope perf \ - 建议用 `--name` 指定有意义的输出目录名（不传 `--model` 时目录名默认取数据集名）。 ``` -## Warmup 预热压测 +## Embedding 与 Rerank -在正式压测前发送一批预热请求，消除冷启动影响（如 KV-cache 填充、JIT 编译、连接池初始化等），使性能指标更准确。 - -预热请求使用与正式压测相同的并发和速率发送，但**不计入性能指标**（延迟、吞吐、百分位等均排除预热数据）。 +### Embedding 模型 -**1. 绝对数量模式** - -指定预热请求的绝对数量： +使用 `openai_embedding` API 模式和 `random_embedding` 数据集进行压测。使用随机数据集时需指定 `tokenizer-path` 用于生成指定长度范围的 query 文本。 ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 10 \ - --model 'qwen2.5' \ - --number 100 \ - --warmup-num 10 \ - --api openai \ - --dataset openqa \ - --stream + --parallel 2 \ + --number 10 \ + --model 'text-embedding-v4' \ + --url 'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings' \ + --api-key ${DASHSCOPE_API_KEY} \ + --api openai_embedding \ + --dataset random_embedding \ + --min-prompt-length 256 \ + --max-prompt-length 256 \ + --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' ``` -上述命令会先发送 10 个预热请求，再发送 100 个正式压测请求，指标仅统计后 100 个请求。 +### Rerank 模型 + +使用 `openai_rerank` API 模式和 `random_rerank` 数据集进行压测。使用随机数据集时需指定 `tokenizer-path` 用于生成指定长度范围的 query 文本。 -**2. 比例模式** +可以通过 `extra-args` 指定生成数据的参数： -使用 0~1 之间的浮点数，按 `--number` 的比例计算预热数量，适用于 sweep 模式（多轮 `--number` 不同时自动适配）： +- `num_documents`：每条 query 对应的文档数量 +- `document_length_ratio`：文档长度相对于 query 长度的倍数 ```bash evalscope perf \ - --url 'http://127.0.0.1:8000/v1/chat/completions' \ - --parallel 10 \ - --model 'qwen2.5' \ - --number 100 \ - --warmup-num 0.1 \ - --api openai \ - --dataset openqa \ - --stream -``` - -`--warmup-num 0.1` 表示预热数量为 `--number` 的 10%，即 `max(1, int(0.1 * 100)) = 10` 个预热请求。 - -```{note} -**注意事项** - -- 预热请求与正式请求使用相同的数据集和请求参数。 -- 预热期间的进度条会单独显示（`Warmup[...]`），完成后自动切换到正式压测进度条（`Processing[...]`）。 -- 多轮对话模式下，`--warmup-num` 表示预热的对话数量（与 `--number` 语义一致），预热对话内的所有 turn 均不计入指标。 + --parallel 2 \ + --number 10 \ + --model 'qwen3-rerank' \ + --url 'https://dashscope.aliyuncs.com/compatible-api/v1/reranks' \ + --api-key ${DASHSCOPE_API_KEY} \ + --api openai_rerank \ + --dataset random_rerank \ + --min-prompt-length 256 \ + --max-prompt-length 256 \ + --tokenizer-path 'Qwen/Qwen3-Embedding-0.6B' \ + --extra-args '{"num_documents": 5, "document_length_ratio": 3}' ``` ## 调试请求 + 使用 `--debug` 选项，我们将输出请求和响应，输出示例如下： -**非`stream`模式输出示例** +**非 `stream` 模式输出示例** ```text 2024-11-27 11:25:34,161 - evalscope - http_client.py - on_request_start - 116 - DEBUG - Starting request: )> @@ -378,7 +438,7 @@ evalscope perf \ 2024-11-27 11:25:38,172 - evalscope - http_client.py - on_response_chunk_received - 140 - DEBUG - Request received: ``` -**`stream`模式输出示例** +**`stream` 模式输出示例** ```text 2024-11-27 20:02:24,760 - evalscope - http_client.py - _handle_stream - 57 - DEBUG - Response recevied: data: {"model":"Qwen2.5-0.5B-Instruct","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"重要的"},"finish_reason":null}],"usage":null} @@ -395,14 +455,18 @@ evalscope perf \ 2024-11-27 20:02:25,113 - evalscope - http_client.py - _handle_stream - 57 - DEBUG - Response recevied: data: [DONE] ``` -## 可视化测试结果 +## 结果可视化 + +### WandB + +请使用如下命令安装 wandb： -### 使用WandB -请使用如下命令安装wandb: ```bash pip install wandb ``` -启动测试前添加如下参数: + +启动测试前添加如下参数： + ```bash --visualizer wandb --name 'name_of_wandb_log' @@ -410,40 +474,44 @@ pip install wandb ![wandb sample](https://modelscope.oss-cn-beijing.aliyuncs.com/resource/wandb_sample.png) +### SwanLab -### 使用SwanLab +请使用如下命令安装 SwanLab： -请使用如下命令安装SwanLab: ```bash pip install swanlab ``` -启动测试前添加如下参数: +启动测试前添加如下参数： + ```bash # 可使用 SWANLAB_PROJ_NAME 环境变量指定项目名称 --visualizer swanlab --name 'name_of_swanlab_log' -``` +``` ![swanlab sample](https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/swanlab.png) +### ClearML + +请使用如下命令安装 ClearML： -### 使用ClearML -请使用如下命令安装ClearML: ```bash pip install clearml ``` -初始化ClearML服务器: +初始化 ClearML 服务器： + ```bash clearml-init ``` -启动测试前添加如下参数: +启动测试前添加如下参数： + ```bash # 可使用 CLEARML_PROJECT_NAME 环境变量指定项目名称 --visualizer clearml --name 'name_of_clearml_task' ``` -![clearml sample](https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/doc/clearml_vis.jpg) \ No newline at end of file +![clearml sample](https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/doc/clearml_vis.jpg) diff --git a/docs/zh/user_guides/stress_test/multi_turn.md b/docs/zh/user_guides/stress_test/multi_turn.md index 5b5b24711..766ff85f4 100644 --- a/docs/zh/user_guides/stress_test/multi_turn.md +++ b/docs/zh/user_guides/stress_test/multi_turn.md @@ -1,50 +1,12 @@ # 多轮对话压测 -多轮对话压测功能允许用户在真实的多轮交互场景下测试模型服务。与普通压测不同，多轮模式会将模型的**真实回复**追加到上下文，后续每轮请求都携带完整的历史对话，从而真实模拟用户与模型的连续对话过程，并能评估服务在上下文不断增长时的延迟、吞吐量及 KV 缓存命中率表现。 +多轮对话压测在真实的连续交互场景下测试模型服务：每轮请求成功后，把模型的**真实回复**追加到上下文，下一轮请求携带完整历史，而非仅发送当前用户消息。由此可评估服务在上下文不断增长时的延迟与吞吐表现。 -## 功能特性 +框架同时基于客户端 token 计数估算每轮请求中历史 token 占总输入 token 的比例，即理论上可被服务端 prefix caching 利用的上限；实际是否命中取决于服务端是否开启并维持了对应缓存。 -- **真实上下文累积**：每轮请求成功后，将模型实际输出追加到对话历史，下一轮请求携带完整历史，而非仅发送当前用户消息。 -- **KV 缓存可命中率估算**：基于客户端 token 计数，估算每轮请求中历史对话 token 占总输入 token 的比例，即理论上可被服务端 prefix caching 利用的上限；实际是否命中取决于服务端是否开启并维持了对应缓存。 -- **多数据集支持**：提供随机合成（`random_multi_turn`）、真实对话（`share_gpt_zh_multi_turn` / `share_gpt_en_multi_turn`）、自定义本地数据（`custom_multi_turn`）、真实 Agent 轨迹（`swe_smith`）和生产 agentic trace 重放（`trie_agentic_coding` / `trie_code_qa` / `trie_office_work`）等数据集。 -- **参数语义一致**：`--number` 为总对话数，`--parallel` 为并发对话数，与普通压测模式语义保持一致。 +`--number` 为总对话数、`--parallel` 为并发对话数，与单轮压测语义保持一致。 -## 参数说明 - -### 多轮专属参数 - -| 参数 | 类型 | 说明 | 默认值 | -|------|------|------|--------| -| `--multi-turn` | `bool` | 启用多轮对话压测模式 | `False` | -| `--min-turns` | `int` | 每个对话最少用户轮数，仅 `random_multi_turn` 使用 | `1` | -| `--max-turns` | `int` | 每个对话最多用户轮数；`random_multi_turn` **必须设置**；ShareGPT / `custom_multi_turn` 等数据集可选，用于截断过长对话；`swe_smith` live 构建时每条对话轮次从 `[min_turns, max_turns]` 随机采样 | `None` | -| `--dataset-offset` | `int` | 跳过数据集前 N 条对话，用于分片测试或避免缓存命中 | `0` | - -### `multi_turn_args`（`swe_smith` 专属参数） - -```{note} -`--multi-turn-args` 已**废弃**，请改用统一的 `--dataset-args` 传入同名参数。旧参数仍可用，其内容会自动折叠进 `--dataset-args`（同名键以 `--dataset-args` 为准）。下表参数键名不变。 -``` - -`swe_smith` 数据集的 live 构建模式支持通过 `--dataset-args` 精细控制对话 token 长度目标。 - -每条对话的轮次数量从 `[--min-turns, --max-turns]` 中随机采样，每轮按以下 token 长度参数填充消息。 - -| 参数 | 类型 | 说明 | 默认值 | -|------|------|------|--------| -| `first_turn_length` | `int` 或 `[int, int]` | 第 1 轮目标 prompt token 数；从原始轨迹中截取恰好达到该长度的消息片段
• 单个整数：固定值，如 `65000`
• 两个整数的列表：`[最小值, 最大值]`，每条对话独立均匀随机采样，如 `[32000, 65000]` | `65000` | -| `subsequent_turn_length` | `int` 或 `[int, int]` | 后续每轮在上一轮基础上新增的目标 token 数；决定每轮 delta 的大小
• 单个整数：固定值
• 两个整数的列表：`[最小值, 最大值]`，每条对话独立均匀随机采样 | `500` | -| `chars_per_token` | `float` | 无 tokenizer 时的字符/token 估算比，用于预过滤原始轨迹 | `3.0` | -| `num_workers` | `int` | **已废弃**，请使用顶层 `--num-workers` 参数。原用于控制 live 构建模式下的并行 worker 数量。 | `4` | - -### 已有参数在多轮模式下的语义 - -| 参数 | 多轮模式下的含义 | -|------|----------------| -| `--number` | 总 **对话数**；所有 worker 合计完成的对话数达到此值后停止 | -| `--parallel` | 同时并发执行的对话数（每个 worker 持有一条对话） | - -## 工作流程 +## 工作原理 1. **加载对话池**：启动时从数据集文件顺序读取，将所有对话预加载到内存，最多加载 `--number` 条（防止大数据集占用过多内存）。 @@ -91,11 +53,46 @@ 7. **指标汇总**：所有 worker 完成后，汇总全部延迟、吞吐量及多轮专属指标（平均上下文轮数、KV 缓存命中率）后输出结果。 -> **注意**：请求成功率低于 100% 时，被中断的对话不会贡献后续轮次的上下文，KV 缓存命中率等指标可能偏低。 +```{note} +请求成功率低于 100% 时，被中断的对话不会贡献后续轮次的上下文，KV 缓存命中率等指标可能偏低。 +``` + +## 参数说明 + +### 多轮专属参数 + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `--multi-turn` | `bool` | 启用多轮对话压测模式 | `False` | +| `--min-turns` | `int` | 每个对话最少用户轮数，仅 `random_multi_turn` 使用 | `1` | +| `--max-turns` | `int` | 每个对话最多用户轮数；`random_multi_turn` **必须设置**；ShareGPT / `custom_multi_turn` 等数据集可选，用于截断过长对话；`swe_smith` live 构建时每条对话轮次从 `[min_turns, max_turns]` 随机采样 | `None` | +| `--dataset-offset` | `int` | 跳过数据集前 N 条对话，用于分片测试或避免缓存命中 | `0` | + +### 通用参数的多轮语义 + +以下参数在单轮与多轮模式下含义不同，多轮模式下统一以**对话**为单位： + +| 参数 | 多轮模式下的含义 | +|------|----------------| +| `--number` | 总**对话数**；所有 worker 合计完成的对话数达到此值后停止 | +| `--parallel` | 同时并发执行的对话数（每个 worker 持有一条对话） | +| `--warmup-num` | 预热的**对话数**；预热对话内的所有 turn 均不计入指标 | +| `--duration` | **软退出**：deadline 到点后不再启动新对话，但已在跑的对话会跑完所有 turn 才退出 | +| `--tokenize-prompt` | **不支持**，会被自动忽略；多轮对话始终以消息列表形式发往 `/v1/chat/completions` | + +`--duration` 采用软退出语义，保证每条对话都是完整的，不会出现被截断的对话拉偏统计。三种上限组合： + +| 命令 | 含义 | +|---|---| +| `--number 50` | 跑 50 个对话就停，不限时 | +| `--duration 300` | 跑 300 秒就停，对话跑多少看运气 | +| `--number 50 --duration 300` | 两个上限，**先到先停**（推荐） | + +副作用：实际 wall_time 可能略大于 `--duration`，多出来的就是 in-flight 对话跑完所需的时间。 ## 数据集 -evalscope 提供以下多轮数据集： +evalscope 提供以下多轮数据集：合成数据（`random_multi_turn`）、真实对话（`share_gpt_*_multi_turn`）、自定义本地数据（`custom_multi_turn`）、真实 Agent 轨迹（`swe_smith`）和生产 agentic trace 重放（`trie_*`）。 ### random_multi_turn @@ -114,9 +111,7 @@ evalscope 提供以下多轮数据集： ] ``` -> **注意**：`--tokenize-prompt` 在多轮模式下不支持，会被自动忽略。多轮对话始终通过 `/v1/chat/completions` 端点以消息列表形式发送。 - -**使用示例**：适用场景：快速评测服务在指定 prompt 长度分布和多轮深度下的性能，无需真实数据集。 +**适用场景**：快速评测服务在指定 prompt 长度分布和多轮深度下的性能，无需真实数据集。 ```bash evalscope perf \ @@ -189,9 +184,11 @@ evalscope perf \ ] ``` -> **说明**：数据集中的参考助手回复仅用于结构完整性，不会被直接发送给模型。运行时 worker 始终将模型的**实际输出**追加到上下文，保证历史准确。 +```{note} +数据集中的参考助手回复仅用于结构完整性，不会被直接发送给模型。运行时 worker 始终将模型的**实际输出**追加到上下文，保证历史准确。 +``` -**使用示例**：适用场景：使用真实用户对话分布评测服务，反映更贴近生产环境的性能。 +**适用场景**：使用真实用户对话分布评测服务，反映更贴近生产环境的性能。 ```bash evalscope perf \ @@ -267,6 +264,7 @@ evalscope perf \ ``` 每行需满足： + - 必须是一个 JSON 数组 - 每个元素必须包含 `role` 和 `content` 字段 - `role` 取值为 `user` 或 `assistant` @@ -282,18 +280,11 @@ evalscope perf \ ] ``` -> **说明**：数据集中的 `assistant` 消息仅用于标识对话结构，**不会**被直接发送给模型。运行时 worker 始终将模型的实际输出追加到上下文，保证历史准确。 - -**使用示例**：适用场景：已有 OpenAI messages 格式的对话数据，直接用于多轮压测，无需转换格式。 - -首先准备 JSONL 数据文件（每行一条对话）： - -```json -[{"role": "user", "content": "你好"}, {"role": "assistant", "content": "你好！有什么可以帮助你？"}, {"role": "user", "content": "帮我写一首诗"}, {"role": "assistant", "content": "好的，..."}, {"role": "user", "content": "再写一首"}] -[{"role": "user", "content": "介绍一下北京"}, {"role": "assistant", "content": "北京是中国的首都..."}, {"role": "user", "content": "有哪些著名景点？"}] +```{note} +数据集中的 `assistant` 消息仅用于标识对话结构，**不会**被直接发送给模型。运行时 worker 始终将模型的实际输出追加到上下文，保证历史准确。 ``` -然后运行压测： +**适用场景**：已有 OpenAI messages 格式的对话数据，直接用于多轮压测，无需转换格式。 ```bash evalscope perf \ @@ -318,16 +309,26 @@ evalscope perf \ - **预构建 JSON 模式**（推荐）：指定 `--dataset-path` 加载已生成的 `agentic_dataset.json`，无需 tokenizer，启动快。 - **Live 构建模式**（不指定 `--dataset-path`）：运行时从 ModelScope 拉取原始轨迹并动态构建对话，**必须**指定 `--tokenizer-path` 用于精确 token 计数。 -两种模式共同特性： -- **支持偏移**：通过 `--dataset-offset` 跳过前 N 条对话，用于分片测试或规避缓存热点 +两种模式均支持 `--dataset-offset` 跳过前 N 条对话，用于分片测试或规避 KV 缓存热点。 -> **说明**：每条对话的轮次数量从 `[--min-turns, --max-turns]` 中随机采样，每轮填充的消息量由 `first_turn_length` / `subsequent_turn_length` 决定。 +**长度控制参数** -**预构建数据集生成** +Live 构建模式下，每条对话的轮次数量从 `[--min-turns, --max-turns]` 中随机采样，每轮按以下 token 长度参数填充消息。通过 `--dataset-args` 传入： + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `first_turn_length` | `int` 或 `[int, int]` | 第 1 轮目标 prompt token 数；从原始轨迹中截取恰好达到该长度的消息片段
• 单个整数：固定值，如 `65000`
• 两个整数的列表：`[最小值, 最大值]`，每条对话独立均匀随机采样，如 `[32000, 65000]` | `65000` | +| `subsequent_turn_length` | `int` 或 `[int, int]` | 后续每轮在上一轮基础上新增的目标 token 数；决定每轮 delta 的大小
• 单个整数：固定值
• 两个整数的列表：`[最小值, 最大值]`，每条对话独立均匀随机采样 | `500` | +| `chars_per_token` | `float` | 无 tokenizer 时的字符/token 估算比，用于预过滤原始轨迹 | `3.0` | +| `num_workers` | `int` | **已废弃**，请使用顶层 `--num-workers` 参数。原用于控制 live 构建模式下的并行 worker 数量。 | `4` | -推荐在压测前使用 `examples/perf/build_swe_smith_dataset.py` 脚本预先构建 `agentic_dataset.json`，一次生成、多次复用，避免每次压测都重复下载和构建。 +```{note} +`--multi-turn-args` 已**废弃**，请改用统一的 `--dataset-args` 传入上表参数。旧参数仍可用，其内容会自动折叠进 `--dataset-args`（同名键以 `--dataset-args` 为准），参数键名不变。 +``` -**主要参数**： +**预构建数据集** + +推荐在压测前使用 `examples/perf/build_swe_smith_dataset.py` 脚本预先构建 `agentic_dataset.json`，一次生成、多次复用，避免每次压测都重复下载和构建。注意脚本参数为 CLI 短横线形式，与上表 `--dataset-args` 中的下划线键名一一对应： | 参数 | 说明 | 默认值 | |------|------|--------| @@ -372,8 +373,6 @@ evalscope perf \ --parallel 20 ``` -> **说明**：`--dataset-offset` 可跳过数据集前 N 条对话，适合多机分片压测或规避 KV 缓存热点。 - **使用示例：Live 构建模式** 自动从 ModelScope 拉取 SWE-smith-trajectories 数据集并实时构建对话，需指定 `--tokenizer-path`： @@ -424,8 +423,6 @@ evalscope perf ... --dataset trie_office_work --dataset-path /path/to/your_trace **必需参数**：`--tokenizer-path`（用于合成精确长度的 prompt） -**使用示例**： - ```bash evalscope perf \ --model qwen-plus \ @@ -442,11 +439,13 @@ evalscope perf \ --stream ``` -> **注意**：trie trace 重放依赖 `ignore_eos` 让模型严格生成到录制长度。vLLM 和 SGLang 支持此参数；DashScope / OpenAI API 等云端服务不支持，模型遇到自然 EOS 就停，实际输出会短于录制长度，但不影响 latency / cache hit / decode TPS 的正确性。 +```{note} +trie trace 重放依赖 `ignore_eos` 让模型严格生成到录制长度。vLLM 和 SGLang 支持此参数；DashScope / OpenAI API 等云端服务不支持，模型遇到自然 EOS 就停，实际输出会短于录制长度，但不影响 latency / cache hit / decode TPS 的正确性。 +``` -## 多轮专属输出指标 +## 输出指标 -除了 Performance Overview 和 Per-Request Metrics 两张表外，多轮模式还会额外输出以下表格。 +除了 Performance Overview 和 Per-Request Metrics 两张表外，多轮模式还会额外输出以下内容。 ### Per-Trace Metrics @@ -463,6 +462,10 @@ evalscope perf \ 同时输出 `trace_summary.json` 到结果目录。 +```{note} +多轮走 `/v1/chat/completions` 端点时，chat template 每轮会额外增加 10-50 个 token（role marker、special token 等），导致 Cache Hit Rate 比 raw completions 接口低 2-3pp，这是正常现象。 +``` + ### Workload Throughput 按时间维度汇总的 token 吞吐率，所有模式（单轮 / 多轮）均输出： @@ -493,31 +496,9 @@ evalscope perf \ Steady-state 是评估「这个 endpoint 稳定状态下能跑多快」的最公平数字；Last 30s 适合看尾部抖动。同时输出 `workload_throughput.json` 和 `workload_timeline.json`（raw cumulative-token 时间序列，pandas 友好）到结果目录。 -### First-Turn vs Subsequent-Turn TTFT +### 首轮与后续轮 TTFT -Per-Request Metrics 表里的 `TTFT (ms)` 是所有 turn 的均值。在多轮场景下，第一 turn 是冷 prefill，后续 turn 大量命中 prefix cache，两类 TTFT 量级可能差 2-10 倍。Per-Request Metrics 表会额外出现： +Per-Request Metrics 表里的 `TTFT (ms)` 是所有 turn 的均值。在多轮场景下，第一 turn 是冷 prefill，后续 turn 大量命中 prefix cache，两类 TTFT 量级可能差 2-10 倍。因此 Per-Request Metrics 表会额外出现两行，只在多轮路径下显示： - `First-Turn TTFT (ms)` - `Subsequent-Turn TTFT (ms)` - -只在多轮路径下出现；单轮压测不显示。 - -## `--duration` 软退出 - -多轮模式下 `--duration` 采用软退出语义：deadline 到点后**不再启动新对话**，但**已经在跑的对话会跑完所有 turn** 才退出。这保证每条对话都是完整的，不会出现被截断的对话拉偏统计。 - -三种 cap 组合： - -| 命令 | 含义 | -|---|---| -| `--number 50` | 跑 50 个对话就停，不限时 | -| `--duration 300` | 跑 300 秒就停，对话跑多少看运气 | -| `--number 50 --duration 300` | 两个上限，**先到先停**（推荐） | - -副作用：实际 wall_time 可能略大于 `--duration`（多出来的就是 in-flight 对话跑完所需的时间）。 - -## 常见问题 - -### Chat template 带来的 token 差异 - -多轮走 `/v1/chat/completions` 端点时，chat template 每轮会额外增加 10-50 个 token（role marker、special token 等），导致 Cache Hit Rate 比 raw completions 接口低 2-3pp，这是正常现象。 diff --git a/docs/zh/user_guides/stress_test/parameters.md b/docs/zh/user_guides/stress_test/parameters.md index 02f128eae..ba36a5c2b 100644 --- a/docs/zh/user_guides/stress_test/parameters.md +++ b/docs/zh/user_guides/stress_test/parameters.md @@ -35,8 +35,8 @@ | `--log-every-n-query` | `int` | 每N个查询记录日志 | `100` | | `--stream` | `bool` | 是否使用SSE流输出
需要启用以测量TTFT（Time to First Token）指标 | `True` | | `--sleep-interval` | `int` | 每次性能测试之间的休眠时间（秒）
避免过载服务器 | `5` | -| `--open-loop` | `bool` | 启用开放环路（open-loop）模式：
请求按 `--rate` 指定的速率发出，无论服务端是否已处理完之前的请求。
• `--rate` 变为扫描变量（支持多值）
• `--number` 须与 `--rate` 等长，表示每轮发出的请求总数
• `--parallel` 在此模式下被忽略（内部设为 -1 / INF）
详见[使用示例](./examples.md#open-loop-开放环路模式) | `False` | -| `--warmup-num` | `float` | 预热请求数量或比例：
• `0`：禁用预热（默认）
• `>= 1`：绝对数量，如 `--warmup-num 10` 表示预热 10 个请求
• `0 < value < 1`：比例模式，如 `--warmup-num 0.1` 表示预热数量为 `--number` 的 10%
预热请求使用与正式压测相同的并发/速率发送，但**不计入性能指标**
适用于消除冷启动影响（如 KV-cache 填充、JIT 编译等）
详见[使用示例](./examples.md#warmup-预热压测) | `0` | +| `--open-loop` | `bool` | 启用开放环路（open-loop）模式：
请求按 `--rate` 指定的速率发出，无论服务端是否已处理完之前的请求。
• `--rate` 变为扫描变量（支持多值）
• `--number` 须与 `--rate` 等长，表示每轮发出的请求总数
• `--parallel` 在此模式下被忽略（内部设为 -1 / INF）
详见[使用示例](./examples.md#open-loop-开放环路) | `False` | +| `--warmup-num` | `float` | 预热请求数量或比例：
• `0`：禁用预热（默认）
• `>= 1`：绝对数量，如 `--warmup-num 10` 表示预热 10 个请求
• `0 < value < 1`：比例模式，如 `--warmup-num 0.1` 表示预热数量为 `--number` 的 10%
预热请求使用与正式压测相同的并发/速率发送，但**不计入性能指标**
适用于消除冷启动影响（如 KV-cache 填充、JIT 编译等）
详见[使用示例](./examples.md#warmup-预热) | `0` | | `--duration` | `float` | 单次压测的墙钟时间预算（秒）
软退出语义：到点后**不再启动新请求**，但**已经在飞行中的请求会跑完**才退出
多轮模式下的"已在飞"指的是**已经 claim 的 trace 跑完所有剩余 turn**（trace-level soft exit，与上游 trie 一致）
与 `--number` 同时设置时取**先达到的那个**为停止条件 | `None` | ```{tip} @@ -67,32 +67,29 @@ SLA自动调优功能使用详见[自动调优指南](./sla_auto_tune.md)。 ``` -## Prompt设置 - -| 参数 | 类型 | 说明 | 默认值 | -|------|------|------|--------| -| `--max-prompt-length` | `int` | 最大输入prompt长度
超过该值时将丢弃prompt | `131072` | -| `--min-prompt-length` | `int` | 最小输入prompt长度
小于该值时将丢弃prompt | `0` | -| `--prefix-length` | `int` | prompt的前缀长度
仅对`random`数据集有效 | `0` | -| `--prompt` | `str` | 指定请求prompt
字符串或本地文件（通过`@/path/to/file`指定）
优先级高于`dataset`
示例：`@./prompt.txt` | - | -| `--query-template` | `str` | 指定查询模板
JSON字符串或本地文件（通过`@/path/to/file`指定）
示例：`@./query_template.json` | - | -| `--apply-chat-template` | `bool` | 是否应用聊天模板 | `None`（根据URL后缀自动判断） | -| `--image-width` | `int` | 随机VL数据集图像宽度 | `224` | -| `--image-height` | `int` | 随机VL数据集图像高度 | `224` | -| `--image-format` | `str` | 随机VL数据集图像格式 | `RGB` | -| `--image-num` | `int` | 随机VL数据集图像数量 | `1` | -| `--image-patch-size` | `int` | 图像的patch大小
仅用于本地图像token计算 | `28` | - ## 数据集配置 | 参数 | 类型 | 说明 | 默认值 | |------|------|------|--------| -| `--dataset` | `str` | 数据集模式，详见下表 | - | +| `--dataset` | `str` | 数据集模式，详见下方[数据集模式](#数据集模式) | - | | `--dataset-path` | `str` | 数据集文件或目录路径
指向文件时直接读取；指向目录时在目录内查找对应数据文件（适用于离线环境使用已下载的数据集缓存） | - | | `--data-source` | `str` | 数据集加载源，可选值：`modelscope`、`huggingface`、`local`
未指定时默认使用 `modelscope`；当 `--dataset-path` 为本地目录时自动视为 `local` | `modelscope` | -| `--dataset-args` | `str` | 数据集专属参数（JSON 字符串），按 `--dataset` 所选数据集的 schema 校验（传入未知键会直接报错）。承载真实文本数据集的定长输入参数 `target_input_len` / `input_len_mode`，并取代已废弃的 `--multi-turn-args`。详见下方「dataset-args：数据集专属参数」小节 | - | +| `--dataset-args` | `str` | 数据集专属参数（JSON 字符串），按 `--dataset` 所选数据集的 schema 校验（传入未知键会直接报错） | - | + +`--dataset-args` 承载的键按用途分布在下列小节： -### dataset 模式说明 +| 键 | 用途 | 所在小节 | +|----|------|----------| +| `target_input_len` / `input_len_mode` | 把真实数据的输入截断到固定长度 | [长度控制](#长度控制) | +| `prefix_file` / `prefix_role` | 长上下文前缀注入，构造超长定长输入 | [长上下文前缀注入](#长上下文前缀注入) | +| `speed` / `model_override` / `model_mapping` / `match_output_length` | 生产流量回放的回放行为 | [生产流量回放](#生产流量回放) | +| 各类 token 长度参数 | 多轮对话数据集 | [多轮对话](#多轮对话) | + +```{note} +`--multi-turn-args` 已废弃，请改用 `--dataset-args`（键名不变）。旧参数仍可用，会自动并入 `--dataset-args`（同名键以 `--dataset-args` 为准）。 +``` + +### 数据集模式 **文本对话类** @@ -101,7 +98,7 @@ SLA自动调优功能使用详见[自动调优指南](./sla_auto_tune.md)。 | `openqa` | 从ModelScope自动下载[OpenQA](https://www.modelscope.cn/datasets/AI-ModelScope/HC3-Chinese/summary)
prompt长度较短（一般<100 token）
指定`dataset_path`时使用jsonl文件的`question`字段 | ✓ | | `longalpaca` | 从ModelScope自动下载[LongAlpaca-12k](https://www.modelscope.cn/datasets/AI-ModelScope/LongAlpaca-12k/dataPeview)
prompt长度较长（一般>6000 token）
指定`dataset_path`时使用jsonl文件的`instruction`字段 | ✓ | | `line_by_line` | 逐行将txt文件的每一行作为一个prompt
**必需提供`dataset_path`** | ✓（必需） | -| `random` | 根据`prefix-length`、`max-prompt-length`和`min-prompt-length`随机生成prompt
**必需指定`tokenizer-path`**
[使用示例](./examples.md#使用random数据集) | ✗ | +| `random` | 根据`prefix-length`、`max-prompt-length`和`min-prompt-length`随机生成prompt
**必需指定`tokenizer-path`**
[使用示例](./examples.md#随机数据集) | ✗ | | `custom` | 自定义数据集解析器
参考[自定义数据集指南](custom.md/#自定义数据集) | ✓ | **多模态类** @@ -110,7 +107,7 @@ SLA自动调优功能使用详见[自动调优指南](./sla_auto_tune.md)。 |------|------|------------------| | `flickr8k` | 从ModelScope自动下载[Flick8k](https://www.modelscope.cn/datasets/clip-benchmark/wds_flickr8k/dataPeview)
构建图文输入，数据集较大，适合评测多模态模型
支持`--dataset-path`指向本地数据集目录（离线环境） | ✓（目录） | | `kontext_bench` | 从ModelScope自动下载[Kontext-Bench](https://modelscope.cn/datasets/black-forest-labs/kontext-bench/dataPeview)
构建图文输入，约1000条数据，适合快速评测多模态模型
支持`--dataset-path`指向本地数据集目录（离线环境） | ✓（目录） | -| `random_vl` | 随机生成图像和文本输入
在`random`基础上增加图像相关参数
[使用示例](./examples.md#使用random图文数据集) | ✗ | +| `random_vl` | 随机生成图像和文本输入
在`random`基础上增加图像相关参数
[使用示例](./examples.md#随机图文数据集) | ✗ | **Embedding 类** @@ -130,7 +127,7 @@ SLA自动调优功能使用详见[自动调优指南](./sla_auto_tune.md)。 **多轮对话类** -需配合 `--multi-turn` 使用，详见[多轮对话压测指南](./multi_turn.md)。 +需配合 `--multi-turn` 使用，参数见[多轮对话](#多轮对话)，详见[多轮对话压测指南](./multi_turn.md)。 | 模式 | 说明 | 支持dataset-path | |------|------|------------------| @@ -141,19 +138,29 @@ SLA自动调优功能使用详见[自动调优指南](./sla_auto_tune.md)。 **生产流量回放类** -需配合 `--open-loop` 使用，按录制的**原始到达节奏**逐字回放真实请求。详见[使用示例](./examples.md#生产流量回放workload_trace)。 +需配合 `--open-loop` 使用，trace 文件格式与回放参数见[生产流量回放](#生产流量回放)。 | 模式 | 说明 | 支持dataset-path | |------|------|------------------| | `workload_trace` | 回放录制的生产流量 JSONL：按原始时间戳、请求体、headers 逐字重放，贴近真实负载（突发流量、异构请求、多模型路由）
每条请求自带 `model`，保留多模型混合路由
**必需 `--open-loop` 和 `--dataset-path`**；`--model`/`--number` 可选（trace 自带模型与条数） | ✓（必需） | -### dataset-args：数据集专属参数 +## 输入构造 -`--dataset-args` 用一个 JSON 字符串为不同数据集传入专属参数（键名写错会直接报错，便于排查）。常见两个场景： +控制送入模型的输入内容与长度。以下小节的 `--xxx` 为命令行参数，无前缀的键通过 `--dataset-args` 的 JSON 传入。 -**场景一：把真实数据的输入截断到固定长度** +### 长度控制 -想用真实数据（而非 `random`）压测某个**固定输入长度**时用它。支持 `openqa`、`longalpaca`、`line_by_line`、单轮 ShareGPT（`share_gpt_zh` / `share_gpt_en`），**需配合 `--tokenizer-path`**。 +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `--max-prompt-length` | `int` | 最大输入prompt长度
超过该值时将丢弃prompt | `131072` | +| `--min-prompt-length` | `int` | 最小输入prompt长度
小于该值时将丢弃prompt | `0` | + +想用真实数据（而非 `random`）压测某个**固定输入长度**时，用 `--dataset-args` 的下列键。支持 `openqa`、`longalpaca`、`line_by_line`（仅纯文本行）、ShareGPT（`share_gpt_zh` / `share_gpt_en`），**需配合 `--tokenizer-path`**。 + +| 键 | 说明 | 默认值 | +|------|------|--------| +| `target_input_len` | 目标输入 token 数。设置后每条 prompt 都会被截断到该长度 | 不启用 | +| `input_len_mode` | 对**短于目标**的 prompt 怎么处理：`cap`（原样保留，该条长度可能小于目标）；`drop`（丢弃，保证产出的每条都恰好等于目标）。`drop` 不能与 `prefix_file` 同用（见[长上下文前缀注入](#长上下文前缀注入)） | `cap` | ```bash # 把每条输入截断到 2048 token @@ -163,10 +170,7 @@ evalscope perf \ --dataset-args '{"target_input_len": 2048}' ``` -| 键 | 说明 | 默认值 | -|------|------|--------| -| `target_input_len` | 目标输入 token 数。设置后每条 prompt 都会被截断到该长度 | 不启用 | -| `input_len_mode` | 对**短于目标**的 prompt 怎么处理：`cap`（原样保留，该条长度可能小于目标）；`drop`（丢弃，保证产出的每条都恰好等于目标） | `cap` | +长度口径为不含 chat template 开销的裸内容 token 数。ShareGPT 等多轮数据集默认只对**最后一轮 user 内容**做截断/过滤（与单轮一致）；仅当配置了 `prefix_file` 时才改为按整段对话所有消息内容之和计量、超长丢弃、不足由前缀补齐（见[长上下文前缀注入](#长上下文前缀注入)）。`line_by_line` 的 JSON 行（messages 数组 / 完整 request body）不走长度控制，同时设置本节参数会直接报错。 它和 `--max/min-prompt-length` 的区别： - `--max/min-prompt-length` 只**筛选**、不改内容——长度不在区间内的样本被丢弃，你得到的是长短不一的真实样本； @@ -174,19 +178,81 @@ evalscope perf \ > 想让“每条恰好 N token”，只能用 `target_input_len`；把 `--min-prompt-length` 和 `--max-prompt-length` 设成相等是做不到的（真实数据几乎没有恰好等于 N 的，会被筛空）。`random` 数据集除外——它是现场生成的，min=max 即可定长，无需本参数。 -**场景二：多轮对话数据集的长度参数** +### 长上下文前缀注入 + +真实指令集大多只有 4K-8K token，直接把 `target_input_len` 设成 128K 会导致 `drop` 模式筛空、`cap` 模式长短不一。`prefix_file` 允许指定一份长文本（如书籍、文档语料），框架按 token 预算把它精确切成 `target_input_len − prompt 长度` 的前缀与短 prompt 拼接，使**每条请求总输入恰好等于目标长度**，且保持真实人类语言的低熵特征（适合测 Prefix-Cache 命中率、MTP 接受率）。适用数据集与[长度控制](#长度控制)相同。 + +| 键 | 说明 | 默认值 | +|------|------|--------| +| `prefix_file` | 长前缀文本文件路径（UTF-8 纯文本）。**必须同时设置 `target_input_len`**，且不能与 `input_len_mode="drop"` 同用 | 不启用 | +| `prefix_role` | 前缀注入角色：`system`（作为开头的 system 消息注入，贴近真实 RAG 流量，推理框架对 system 的 Prefix-Cache 管理通常有专门优化）；`user`（直接拼在 user 消息内容最前面） | `system` | + +```bash +# 用长文本前缀把每条请求精确对齐到 131072 token，前缀注入 system 角色 +evalscope perf \ + --model qwen2.5 --url http://127.0.0.1:8000/v1/chat/completions \ + --dataset openqa --tokenizer-path /path/to/tokenizer \ + --dataset-args '{"target_input_len": 131072, "prefix_file": "/path/to/long_text.txt", "prefix_role": "system"}' +``` + +行为说明： +- **预算分配**：prompt 保持原样（超长时按 `input_len_mode` 截断），前缀精确切到 `target_input_len − 所有消息内容 token 数`，总长恰为目标值；多轮对话的历史一并计入（见[长度控制](#长度控制)的长度口径）。 +- **与 `drop` 互斥**：`drop` 只保留已经填满 `target_input_len` 的 prompt，前缀预算恒为 0，注入必然失效，因此配置时直接报错。要定长请用 `cap` + 前缀补齐。 +- **前缀不足**：前缀文件 token 数不足以填满剩余预算时，会循环重复（tile）补齐后再精确截断，并打 warning 提示。 +- **降级规则**：`apply_chat_template` 关闭（如 `/v1/completions` 端点）时无法注入 system 消息，自动降级为纯文本前缀拼接并打 warning。 +- **拼接边界**：`prefix_role="user"` 和纯文本降级模式下前缀与 prompt 直接相接，前缀与 prompt 的 token 数是分别计算的，拼接处两侧字符可能被 tokenizer 合并或拆分，因此实测总长可能与目标相差约 ±1 token。`prefix_role="system"`（chat template 模式）有消息标记隔断边界，不受此影响，始终精确。 +- **缓存友好**：所有请求共享同一段前缀开头（长度随各条 prompt 略有差异），天然适配 Prefix-Cache 命中测试。 + +```{note} +本节的 `prefix_file` 注入的是**真实文本**前缀，用于真实数据集；`--prefix-length` 注入的是**随机 token** 前缀，只对 `random` 数据集有效（见下方[随机数据生成](#随机数据生成)）。两者用途不同，不要混用。 +``` + +### 随机数据生成 -`swe_smith` 等多轮数据集的 token 长度参数也通过 `--dataset-args` 传入，详见[多轮对话压测指南](./multi_turn.md)。 +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `--prefix-length` | `int` | prompt 的随机 token 前缀长度
仅对 `random` 数据集有效；所有请求共享同一前缀，可用于制造 Prefix-Cache 命中 | `0` | +| `--image-width` | `int` | 随机VL数据集图像宽度 | `224` | +| `--image-height` | `int` | 随机VL数据集图像高度 | `224` | +| `--image-format` | `str` | 随机VL数据集图像格式 | `RGB` | +| `--image-num` | `int` | 随机VL数据集图像数量 | `1` | +| `--image-patch-size` | `int` | 图像的patch大小
仅用于本地图像token计算 | `28` | + +`random` 数据集的长度由 `--min-prompt-length` / `--max-prompt-length` 决定（两者相等即定长），无需 `target_input_len`。 + +### Prompt 与模板 -**场景三：回放真实生产流量** +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `--prompt` | `str` | 指定请求prompt
字符串或本地文件（通过`@/path/to/file`指定）
优先级高于`dataset`
示例：`@./prompt.txt` | - | +| `--query-template` | `str` | 指定查询模板
JSON字符串或本地文件（通过`@/path/to/file`指定）
示例：`@./query_template.json` | - | +| `--apply-chat-template` | `bool` | 是否应用聊天模板 | `None`（根据URL后缀自动判断） | +| `--tokenize-prompt` | `bool` | 在客户端将prompt tokenize为token ID列表，绕过服务端重新tokenize，通过`/v1/completions`直接发送 | `False` | + +## 多轮对话 + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `--multi-turn` | `bool` | 启用多轮对话压测模式；`--number` 表示总发送 turn 数，`--parallel` 表示并发 turn 数 | `False` | +| `--min-turns` | `int` | 每个对话最少用户轮数；`random_multi_turn` 与 `swe_smith` 使用 | `1` | +| `--max-turns` | `int` | 每个对话最多用户轮数；`random_multi_turn` 必需；ShareGPT/`custom_multi_turn` 可选（截断过长对话）；`swe_smith` 用作每个对话轮数采样上界，未设置时回退到 `--min-turns` | `None` | +| `--num-workers` | `int` | CPU 密集型数据集/请求生成的 worker 进程数。
`0` = 根据 CPU 亲和性自动检测；`1` = 串行（无多进程）；`>1` = 显式指定 worker 数。
用于 `random`（长 prompt 并行生成）和 `swe_smith`（live 构建）。取代已废弃的 `multi_turn_args.num_workers`。 | `0` | -用 `workload_trace` 把录制的生产流量按**原始到达节奏**逐字回放，贴近真实负载。**必需 `--open-loop`**，无需 `--rate`（到达时刻由 trace 时间戳决定）。完整示例见[生产流量回放（workload_trace）](./examples.md#生产流量回放workload_trace)。 +`swe_smith` 等多轮数据集的 token 长度参数通过 `--dataset-args` 传入。 + +```{seealso} +可用的多轮数据集见[数据集模式](#数据集模式)，完整用法详见[多轮对话压测指南](./multi_turn.md)。 +``` + +## 生产流量回放 + +用 `--dataset workload_trace` 把录制的生产流量按**原始到达节奏**逐字回放，贴近真实负载。**必需 `--open-loop`**，无需 `--rate`（到达时刻由 trace 时间戳决定）。完整示例见[生产流量回放](./examples.md#生产流量回放)。 trace 文件为 JSONL，每行一条请求记录： -```jsonl +```json {"body": {"model": "qwen-plus", "messages": [{"role": "user", "content": "hi"}]}, "timestamp": 1700000000.0} -{"body": {"model": "qwen-max", "messages": [...]}, "timestamp": 1700000001.5, "headers": {"X-Tag": "exp"}, "request_id": "req-42", "completion_tokens": 256} +{"body": {"model": "qwen-max", "messages": [{"role": "user", "content": "hello"}]}, "timestamp": 1700000001.5, "headers": {"X-Tag": "exp"}, "request_id": "req-42", "completion_tokens": 256} ``` | 字段 | 必需 | 说明 | @@ -197,6 +263,8 @@ trace 文件为 JSONL，每行一条请求记录： | `request_id` | | 透传到结果，用于与原始请求关联 | | `completion_tokens` | | 配合 `match_output_length` 使用 | +回放行为通过 `--dataset-args` 调整： + | 键 | 类型 | 说明 | 默认值 | |----|------|------|--------| | `speed` | float | 回放倍速（2.0 = 2× 快，0.5 = 2× 慢） | `1.0` | @@ -208,11 +276,7 @@ trace 文件为 JSONL，每行一条请求记录： `--model` 对 `workload_trace` **不会改写** trace body——每条请求保留自己的 `model`，从而保留多模型混合路由。需要改模型请用 `model_override` / `model_mapping`。 ``` -```{note} -`--multi-turn-args` 已废弃，请改用 `--dataset-args`（键名不变）。旧参数仍可用，会自动并入 `--dataset-args`（同名键以 `--dataset-args` 为准）。 -``` - -## 模型设置 +## 模型与生成参数 | 参数 | 类型 | 说明 | 默认值 | |------|------|------|--------| @@ -229,9 +293,8 @@ trace 文件为 JSONL，每行一条请求记录： | `--top-p` | `float` | top_p采样 | - | | `--top-k` | `int` | top_k采样 | - | | `--extra-args` | `str` | 额外传入请求体的参数
JSON字符串格式
示例：`'{"ignore_eos": true}'` | - | -| `--tokenize-prompt` | `bool` | 在客户端将prompt tokenize为token ID列表，绕过服务端重新tokenize，通过`/v1/completions`直接发送 | `False` | -## 数据存储 +## 结果输出 | 参数 | 类型 | 说明 | 默认值 | |------|------|------|--------| @@ -242,19 +305,6 @@ trace 文件为 JSONL，每行一条请求记录： | `--outputs-dir` | `str` | 输出文件路径 | `./outputs` | | `--no-timestamp` | `bool` | 输出目录不包含时间戳 | `False` | -## 多轮对话设置 - -| 参数 | 类型 | 说明 | 默认值 | -|------|------|------|--------| -| `--multi-turn` | `bool` | 启用多轮对话压测模式；`--number` 表示总发送 turn 数，`--parallel` 表示并发 turn 数 | `False` | -| `--min-turns` | `int` | 每个对话最少用户轮数；`random_multi_turn` 与 `swe_smith` 使用 | `1` | -| `--max-turns` | `int` | 每个对话最多用户轮数；`random_multi_turn` 必需；ShareGPT/`custom_multi_turn` 可选（截断过长对话）；`swe_smith` 用作每个对话轮数采样上界，未设置时回退到 `--min-turns` | `None` | -| `--num-workers` | `int` | CPU 密集型数据集/请求生成的 worker 进程数。
`0` = 根据 CPU 亲和性自动检测；`1` = 串行（无多进程）；`>1` = 显式指定 worker 数。
用于 `random`（长 prompt 并行生成）和 `swe_smith`（live 构建）。取代已废弃的 `multi_turn_args.num_workers`。 | `0` | - -```{seealso} -多轮对话压测使用详见[多轮对话压测指南](./multi_turn.md)。 -``` - ## 其他参数 | 参数 | 类型 | 说明 | 默认值 | diff --git a/docs/zh/user_guides/stress_test/quick_start.md b/docs/zh/user_guides/stress_test/quick_start.md index a000fb7e5..ba1f812d5 100644 --- a/docs/zh/user_guides/stress_test/quick_start.md +++ b/docs/zh/user_guides/stress_test/quick_start.md @@ -79,7 +79,7 @@ results = run_perf_benchmark(task_cfg) - `url`: 请求的URL地址 - `model`: 使用的模型名称 - `api`: 使用的API服务，默认为`openai` -- `dataset`: 数据集名称，此处为`random`，表示随机生成数据集，具体使用说明[参考](./examples.md#使用random数据集)；更多可用的(多模态)数据集请参考[数据集配置](./parameters.md#数据集配置) +- `dataset`: 数据集名称，此处为`random`，表示随机生成数据集，具体使用说明[参考](./examples.md#随机数据集)；更多可用的(多模态)数据集请参考[数据集配置](./parameters.md#数据集配置) - `tokenizer-path`: 模型的tokenizer路径，用于计算token数量（在random数据集中是必须的） - `extra-args`: 请求中的额外的参数，传入json格式的字符串，例如`{"ignore_eos": true}`表示忽略结束token @@ -204,7 +204,7 @@ Percentile results: #### Per-Trace 与 Workload Throughput 指标 -多轮对话模式下还会输出 **Per-Trace Metrics**（对话级分布指标）和 **Workload Throughput**（全局时间维度token吞吐率）两张表，详细说明请参考[多轮对话压测 — 多轮专属输出指标](./multi_turn.md#多轮专属输出指标)。 +多轮对话模式下还会输出 **Per-Trace Metrics**（对话级分布指标）和 **Workload Throughput**（全局时间维度token吞吐率）两张表，详细说明请参考[多轮对话压测 — 输出指标](./multi_turn.md#输出指标)。 ## 可视化测试结果 diff --git a/evalscope/perf/plugin/datasets/base.py b/evalscope/perf/plugin/datasets/base.py index 575e559fc..764da6c20 100644 --- a/evalscope/perf/plugin/datasets/base.py +++ b/evalscope/perf/plugin/datasets/base.py @@ -8,7 +8,15 @@ from evalscope.constants import HubType from evalscope.perf.arguments import Arguments from evalscope.perf.plugin.datasets.dataset_args import BaseDatasetArgs, TextLengthArgs -from evalscope.perf.plugin.datasets.utils import fit_text_to_token_len, load_tokenizer, tokenize_chat_messages +from evalscope.perf.plugin.datasets.utils import ( + fit_prefix_to_budget, + fit_text_to_token_len, + load_tokenizer, + tokenize_chat_messages, +) +from evalscope.utils.logger import get_logger + +logger = get_logger() Message = Dict[str, Any] # single OpenAI message: {"role": ..., "content": ...} Messages = List[Message] # delta messages for one turn @@ -70,6 +78,10 @@ def __init__(self, query_parameters: Arguments): and self.tokenizer is None ): raise ValueError('`target_input_len` requires a tokenizer; please set --tokenizer-path.') + self._prefix_ids: Optional[List[int]] = None + if isinstance(self.dataset_args, TextLengthArgs) and self.dataset_args.prefix_file is not None: + self._prefix_ids = self._load_prefix_ids(self.dataset_args.prefix_file) + self._warned_conversation_dropped: bool = False def __next__(self): for item in self.build_messages(): @@ -200,6 +212,122 @@ def prepare_prompt(self, prompt: str) -> Optional[str]: raise ValueError('`target_input_len` requires a tokenizer; please set --tokenizer-path.') return fit_text_to_token_len(prompt, args.target_input_len, args.input_len_mode, self.tokenizer) + def _load_prefix_ids(self, prefix_file: str) -> List[int]: + """Read and tokenize the long-context prefix file once at construction time. + + Emits one-shot warnings for the tiling and no-chat-template downgrade + cases so per-request generation stays silent. + """ + args = self.dataset_args + if not os.path.isfile(prefix_file): + raise FileNotFoundError(f"The specified prefix_file '{prefix_file}' does not exist.") + with open(prefix_file, 'r', encoding='utf-8') as f: + prefix_text = f.read() + ids = self.tokenizer.encode(prefix_text, add_special_tokens=False) + if not ids: + raise ValueError(f"The specified prefix_file '{prefix_file}' is empty.") + if len(ids) < args.target_input_len: + logger.warning( + f'prefix_file has {len(ids)} tokens, fewer than target_input_len={args.target_input_len}; ' + 'the prefix will be repeated (tiled) to fill the remaining budget when needed.' + ) + if args.prefix_role == 'system' and not self.query_parameters.apply_chat_template: + logger.warning( + "prefix_role='system' requires a chat template; falling back to plain-text " + 'prefix concatenation because apply_chat_template is disabled.' + ) + return ids + + def content_token_len(self, content: str) -> int: + """Return the bare content token count of one message (no special tokens).""" + return len(self.tokenizer.encode(content, add_special_tokens=False)) + + def measure_messages_len(self, messages: Messages) -> int: + """Return the total bare content token count of a messages list. + + Every message counts, so multi-turn history is included in the length + measurement instead of only the last user turn. + """ + return sum(self.content_token_len(message['content']) for message in messages) + + def apply_prefix_to_messages(self, messages: Messages) -> Messages: + """Inject the configured long-context prefix into an OpenAI messages list. + + The prefix budget is ``target_input_len`` minus the bare content tokens + of **all** messages, so prefix plus conversation together occupy the + target length. ``prefix_role='system'`` prepends a system message + (chat-template mode only), whose content is separated from the prompt by + template markers; otherwise the prefix is prepended to the first message + content so it stays at the very front of the token stream (prefix-cache + friendly). The prefix and prompt are counted independently, so on BPE + tokenizers the total may differ from the target by ~1 token when + characters merge across the join (chat-template mode is exact). No-op + when ``prefix_file`` is not configured or the remaining budget is zero. + """ + if self._prefix_ids is None: + return messages + budget = self.dataset_args.target_input_len - self.measure_messages_len(messages) + if budget <= 0: + return messages + prefix = fit_prefix_to_budget(self._prefix_ids, budget, self.tokenizer) + if not prefix: + return messages + if self.dataset_args.prefix_role == 'system' and self.query_parameters.apply_chat_template: + return [self.create_message(prefix, role='system')] + messages + messages[0]['content'] = prefix + messages[0]['content'] + return messages + + def prepare_messages(self, prompt: str) -> Optional[Union[str, Messages]]: + """Apply length policy, prefix injection and chat wrapping to one prompt. + + Combines :meth:`prepare_prompt` (cap/drop fitting) with the long-context + prefix injection (issue #1524). Returns ``None`` when the prompt should + be skipped, a plain string when ``apply_chat_template`` is off, + otherwise an OpenAI messages list. + """ + prepared = self.prepare_prompt(prompt) + if prepared is None: + return None + if not self.query_parameters.apply_chat_template: + if self._prefix_ids is None: + return prepared + # Plain-text endpoint: fill the remaining budget with the prefix. The + # prefix and prompt are counted independently, so the total may drift + # by ~1 token when characters merge across the join. + budget = self.dataset_args.target_input_len - self.content_token_len(prepared) + return fit_prefix_to_budget(self._prefix_ids, budget, self.tokenizer) + prepared + return self.apply_prefix_to_messages([self.create_message(prepared)]) + + def prepare_conversation(self, messages: Messages) -> Optional[Messages]: + """Fit a whole conversation to ``target_input_len`` by prepending the prefix. + + Called only when a ``prefix_file`` is configured (which requires + ``target_input_len``). The length is measured over **every** message + content, so multi-turn history counts towards the budget. A conversation + already exceeding the target is dropped rather than truncated: cutting the + history or the last user turn would either destroy the dialogue or + silently change what is being benchmarked. Shorter conversations are + filled up by the prefix. + + Args: + messages (Messages): The conversation, ending with a user message. + + Returns: + Optional[Messages]: The adjusted conversation, or ``None`` to skip it. + """ + target_input_len = self.dataset_args.target_input_len + total_len = self.measure_messages_len(messages) + if total_len > target_input_len: + if not self._warned_conversation_dropped: + self._warned_conversation_dropped = True + logger.warning( + f'Dropping conversations whose total content length exceeds target_input_len=' + f'{target_input_len} (first one had {total_len} tokens); the length is measured ' + 'over all turns, so long histories are skipped instead of truncated.' + ) + return None + return self.apply_prefix_to_messages(messages) + def check_prompt_length(self, prompt: str) -> Tuple[bool, int]: """Check if the prompt length is within the specified range. diff --git a/evalscope/perf/plugin/datasets/dataset_args.py b/evalscope/perf/plugin/datasets/dataset_args.py index 99bf43915..cf634ac6d 100644 --- a/evalscope/perf/plugin/datasets/dataset_args.py +++ b/evalscope/perf/plugin/datasets/dataset_args.py @@ -12,7 +12,7 @@ safe because it has no such dependency. """ -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from typing import Literal, Optional from evalscope.perf.multi_turn_args import MultiTurnArgs @@ -49,6 +49,26 @@ class TextLengthArgs(BaseDatasetArgs): yielded prompt is exactly ``target_input_len`` (drops data). """ + prefix_file: Optional[str] = None + """Path to a UTF-8 plain-text file used as a long-context prefix (issue #1524). + + When set, the file content is tokenized once and, for every request, sliced + to exactly ``target_input_len - len(prompt_tokens)`` tokens so the total + input hits the target length with real low-entropy text. If the file is + shorter than the remaining budget it is repeated (tiled) to cover it. + Requires ``target_input_len`` and is incompatible with + ``input_len_mode='drop'``. + """ + + prefix_role: Literal['system', 'user'] = 'system' + """Role used to inject the long prefix. + + - ``system`` (default): the prefix goes into a leading system message, + matching real RAG traffic and inference-engine prefix-cache handling. + Falls back to plain-text concatenation when no chat template is applied. + - ``user``: the prefix is prepended to the (first) user message content. + """ + @field_validator('target_input_len') @classmethod def _validate_target_input_len(cls, v: Optional[int]) -> Optional[int]: @@ -56,6 +76,20 @@ def _validate_target_input_len(cls, v: Optional[int]) -> Optional[int]: raise ValueError(f'target_input_len must be > 0, got {v}') return v + @model_validator(mode='after') + def _validate_prefix_file(self) -> 'TextLengthArgs': + if self.prefix_file is not None and self.target_input_len is None: + raise ValueError('`prefix_file` requires `target_input_len` to be set.') + # `drop` already forces every prompt to exactly `target_input_len`, leaving a + # zero prefix budget, so the prefix would silently never be injected. + if self.prefix_file is not None and self.input_len_mode == 'drop': + raise ValueError( + "`prefix_file` is incompatible with `input_len_mode='drop'`: `drop` keeps only prompts that " + "already fill `target_input_len`, so no prefix budget is left. Use `input_len_mode='cap'` " + 'and let the prefix fill the remaining budget.' + ) + return self + class TextDatasetArgs(TextLengthArgs): """Args schema for single-turn real-text datasets. diff --git a/evalscope/perf/plugin/datasets/line_by_line.py b/evalscope/perf/plugin/datasets/line_by_line.py index 89f9392c3..f38eb5d15 100644 --- a/evalscope/perf/plugin/datasets/line_by_line.py +++ b/evalscope/perf/plugin/datasets/line_by_line.py @@ -37,6 +37,9 @@ class LineByLineDatasetPlugin(DatasetPluginBase): ``__compose_query_from_parameter`` only fills in generation parameters that are **missing** from the body (``setdefault`` semantics), so CLI-level defaults do not silently overwrite a user-supplied body. + + Input-length control (``target_input_len`` / ``prefix_file``) only applies to + format 1; a JSON line raises an error when those arguments are set. """ args_schema = TextDatasetArgs @@ -67,13 +70,17 @@ def build_messages(self) -> Iterator[Union[str, List[Dict], Dict[str, Any]]]: parsed = self._try_parse_json(line) if isinstance(parsed, str): - prompt = self.prepare_prompt(parsed) - if prompt is None: + result = self.prepare_messages(parsed) + if result is None: continue - if self.query_parameters.apply_chat_template: - result = [self.create_message(prompt)] - else: - result = prompt else: + # JSON lines are forwarded verbatim, bypassing prepare_messages, so + # length control would silently not apply and mix input lengths. + if self.dataset_args.target_input_len is not None: + raise ValueError( + '`target_input_len` / `prefix_file` only support plain-text lines in the ' + f'`line_by_line` dataset, but this line is JSON: {line[:80]}... Either drop ' + 'those arguments or use a plain-text dataset file.' + ) result = parsed yield result diff --git a/evalscope/perf/plugin/datasets/longalpaca.py b/evalscope/perf/plugin/datasets/longalpaca.py index 2a424af97..0cc701fb4 100644 --- a/evalscope/perf/plugin/datasets/longalpaca.py +++ b/evalscope/perf/plugin/datasets/longalpaca.py @@ -25,10 +25,7 @@ def build_messages(self) -> Iterator[List[Dict]]: ds = self.load_hub_dataset(dataset_id='AI-ModelScope/LongAlpaca-12k', split='train') for item in ds: prompt = item['instruction'].strip() - prompt = self.prepare_prompt(prompt) - if prompt is None: + result = self.prepare_messages(prompt) + if result is None: continue - if self.query_parameters.apply_chat_template: - yield [self.create_message(prompt)] - else: - yield prompt + yield result diff --git a/evalscope/perf/plugin/datasets/openqa.py b/evalscope/perf/plugin/datasets/openqa.py index 0fbbd3750..aba959ad2 100644 --- a/evalscope/perf/plugin/datasets/openqa.py +++ b/evalscope/perf/plugin/datasets/openqa.py @@ -28,10 +28,7 @@ def build_messages(self) -> Iterator[List[Dict]]: for item in self.dataset_line_by_line(self.query_parameters.dataset_path): item = json.loads(item) prompt = item['question'].strip() - prompt = self.prepare_prompt(prompt) - if prompt is None: + result = self.prepare_messages(prompt) + if result is None: continue - if self.query_parameters.apply_chat_template: - yield [self.create_message(prompt)] - else: - yield prompt + yield result diff --git a/evalscope/perf/plugin/datasets/share_gpt.py b/evalscope/perf/plugin/datasets/share_gpt.py index cdfb634e8..3ea4d205f 100644 --- a/evalscope/perf/plugin/datasets/share_gpt.py +++ b/evalscope/perf/plugin/datasets/share_gpt.py @@ -30,6 +30,12 @@ class ShareGPTDatasetPluginBase(DatasetPluginBase): ] Dataset: https://www.modelscope.cn/datasets/swift/sharegpt + + Length control: with a ``prefix_file`` the budget is measured over the whole + conversation (conversations longer than the target are skipped, shorter ones + padded by the prefix, no turn is truncated). Without a prefix, only the last + user turn is fit / filtered (``target_input_len`` cap/drop or the legacy + ``min/max_prompt_length`` filter), matching the pre-prefix behavior. """ # Subclasses must set this @@ -79,13 +85,19 @@ def build_messages(self) -> Iterator[List[Dict]]: if not messages: continue - # Fit / filter using the last user turn content. - last_user_content = messages[-1]['content'] - prepared = self.prepare_prompt(last_user_content) - if prepared is None: - continue - messages[-1]['content'] = prepared - yield messages + if self._prefix_ids is not None: + # Prefix injection: budget measured over the whole conversation. + prepared = self.prepare_conversation(messages) + if prepared is None: + continue + yield prepared + else: + # Legacy behavior: fit / filter on the last user turn only. + last = self.prepare_prompt(messages[-1]['content']) + if last is None: + continue + messages[-1]['content'] = last + yield messages @register_dataset('share_gpt_zh') diff --git a/evalscope/perf/plugin/datasets/utils.py b/evalscope/perf/plugin/datasets/utils.py index ae3d5c924..d11a38d46 100644 --- a/evalscope/perf/plugin/datasets/utils.py +++ b/evalscope/perf/plugin/datasets/utils.py @@ -1,5 +1,5 @@ import numpy as np -from typing import Dict, List, Optional, Tuple +from typing import Callable, Dict, List, Optional, Tuple from evalscope.utils.logger import get_logger @@ -81,6 +81,52 @@ def tokenize_chat_messages(tokenizer, messages: List[Dict], add_generation_promp ) +def converge_to_token_len( + tokenizer, + token_ids: List[int], + target_len: int, + fill: Callable[[int], List[int]], + add_special_tokens: bool = False, + skip_special_tokens: bool = False, + max_retry: int = 10, +) -> Tuple[str, List[int], int]: + """Decode/re-encode ``token_ids`` until the re-encoded length matches ``target_len``. + + Tokenizers do not guarantee that decoding then re-encoding a token sequence + preserves its length. For example, with GPT2Tokenizer: + ``[6880, 6881] -> ['Ġcalls', 'here'] -> [1650, 939, 486] -> ['Ġcall', 'sh', 'ere']``. + This helper iteratively truncates over-length re-encodes and tops up + under-length ones until the target is hit or ``max_retry`` is exhausted. + + The token source used for topping up is supplied by the caller via ``fill``, + which is what distinguishes the two use cases: random prompt generation + samples arbitrary vocabulary tokens, while long-context prefix injection + continues deterministically through real text. + + Args: + tokenizer: A HuggingFace / ModelScope tokenizer instance. + token_ids: Initial token IDs to converge. + target_len: Desired token count after decode/re-encode. + fill: Returns ``n`` extra token IDs to append when the sequence is short. + add_special_tokens: Whether to add special tokens when re-encoding. + skip_special_tokens: Whether to skip special tokens when decoding. + max_retry: Maximum decode/re-encode refinement rounds. + + Returns: + Tuple of the final text, its re-encoded token IDs, and the token + mismatch (``len(ids) - target_len``, ``0`` when converged). + """ + for attempt in range(max_retry + 1): + text = tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens) + token_ids = tokenizer.encode(text, add_special_tokens=add_special_tokens) + mismatch = len(token_ids) - target_len + # Report the mismatch on the final round instead of adjusting again, so + # the returned text / ids / mismatch stay consistent with each other. + if mismatch == 0 or attempt == max_retry: + return text, token_ids, mismatch + token_ids = token_ids[:target_len] if mismatch > 0 else token_ids + fill(-mismatch) + + def gen_prompt_decode_to_target_len( tokenizer, token_sequence: List[int], @@ -92,51 +138,31 @@ def gen_prompt_decode_to_target_len( """ Ensure decoded-then-encoded prompt length matches the target token length. - This function decodes an initial token sequence to text and re-encodes it, - iteratively adjusting the token sequence length to match a target. - This is necessary because some tokenizers do not guarantee a 1:1 mapping - between consecutive tokens and the decoded-then-encoded sequence length. - For example, for GPT2Tokenizer: - [6880, 6881] -> ['Ġcalls', 'here'] -> - [1650, 939, 486] -> ['Ġcall', 'sh', 'ere'] + Thin wrapper over :func:`converge_to_token_len` that fills length gaps with + random tokens drawn from the allowed (non-special) vocabulary, used for + synthetic ``random`` dataset prompts. Returns a tuple of the final prompt string, adjusted token sequence, and token mismatch. """ - remain_num_try = max_retry - token_mismatch = 0 - vocab_size = len(tokenizer) - # Build the pool of tokens to use when filling gaps; exclude special tokens if possible if allowed_tokens is None: + vocab_size = len(tokenizer) prohibited = set(tokenizer.all_special_ids) allowed_tokens = np.array([t for t in range(vocab_size) if t not in prohibited]) if len(allowed_tokens) == 0: allowed_tokens = np.arange(vocab_size) - while True: - prompt = tokenizer.decode(token_sequence) - token_sequence = tokenizer.encode(prompt, add_special_tokens=add_special_tokens) - - if remain_num_try <= 0: - if len(token_sequence) != target_token_len: - token_mismatch = len(token_sequence) - target_token_len - break + def fill(size: int) -> List[int]: + return allowed_tokens[np.random.randint(0, len(allowed_tokens), size=size)].tolist() - if len(token_sequence) == target_token_len: - break - elif len(token_sequence) < target_token_len: - # Generate extra tokens to reach target length, only from allowed (non-special) tokens - fill_size = target_token_len - len(token_sequence) - indices = np.random.randint(0, len(allowed_tokens), size=fill_size) - extra_tokens = allowed_tokens[indices].tolist() - token_sequence.extend(extra_tokens) - elif len(token_sequence) > target_token_len: - # Truncate to target length - token_sequence = token_sequence[:target_token_len] - - remain_num_try -= 1 - - return prompt, token_sequence, token_mismatch + return converge_to_token_len( + tokenizer=tokenizer, + token_ids=token_sequence, + target_len=target_token_len, + fill=fill, + add_special_tokens=add_special_tokens, + max_retry=max_retry, + ) def truncate_text_to_token_len(text: str, target_len: int, tokenizer, add_special_tokens: bool = False) -> str: @@ -199,3 +225,63 @@ def fit_text_to_token_len( if mode == 'drop': return None raise ValueError(f"Unknown input_len_mode: {mode!r}. Expected one of 'cap', 'drop'.") + + +def fit_prefix_to_budget(prefix_ids: List[int], budget: int, tokenizer, max_retry: int = 10) -> str: + """Slice (tiling if needed) ``prefix_ids`` into text of exactly ``budget`` tokens. + + Used by the long-context prefix injection mechanism (issue #1524): the + prefix must fill the remaining token budget ``target_input_len - prompt_len`` + so the total request input hits the target length. When ``prefix_ids`` is + shorter than the budget it is repeated (tiled) to cover it. + + Thin wrapper over :func:`converge_to_token_len` that fills length gaps by + continuing deterministically through the tiled prefix pool, so the result + stays real low-entropy text (unlike the random filler used for synthetic + prompts). + + Args: + prefix_ids: Token IDs of the full prefix text (``add_special_tokens=False``). + budget: Target token count; ``<= 0`` returns an empty string. + tokenizer: A HuggingFace / ModelScope tokenizer instance. + max_retry: Maximum decode/re-encode refinement rounds. + + Returns: + The prefix text occupying (as close as possible to) ``budget`` tokens. + + Raises: + ValueError: If ``prefix_ids`` is empty while ``budget`` is positive. + """ + if budget <= 0: + return '' + if not prefix_ids: + raise ValueError('prefix_ids must not be empty when budget > 0.') + + pool = list(prefix_ids) + while len(pool) < budget: + pool.extend(prefix_ids) + cursor = budget + + def fill(size: int) -> List[int]: + """Take the next unconsumed slice of the pool, extending it on demand.""" + nonlocal cursor + while cursor + size > len(pool): + pool.extend(prefix_ids) + chunk = pool[cursor:cursor + size] + cursor += size + return chunk + + text, _, mismatch = converge_to_token_len( + tokenizer=tokenizer, + token_ids=pool[:budget], + target_len=budget, + fill=fill, + skip_special_tokens=True, + max_retry=max_retry, + ) + if mismatch != 0: + logger.warning( + f'fit_prefix_to_budget: prefix converged to {budget + mismatch} tokens instead of the ' + f'{budget}-token budget after {max_retry} retries (tokenizer round-trip drift).' + ) + return text diff --git a/tests/perf/test_prefix_injection.py b/tests/perf/test_prefix_injection.py new file mode 100644 index 000000000..980b8a793 --- /dev/null +++ b/tests/perf/test_prefix_injection.py @@ -0,0 +1,315 @@ +"""End-to-end tests for long-context prefix injection (issue #1524). + +Every case drives the real pipeline (``Arguments`` -> plugin -> ``build_messages``) +with char-based fake tokenizers, so the budget arithmetic, message assembly and +config guardrails are all exercised through the public path without needing a +network or a real model. +""" + +import json +import pytest +from pydantic import ValidationError + +from evalscope.perf.arguments import Arguments +from evalscope.perf.plugin.datasets import base as base_mod +from evalscope.perf.plugin.datasets.line_by_line import LineByLineDatasetPlugin +from evalscope.perf.plugin.datasets.share_gpt import ShareGPTZhDatasetPlugin + +PREFIX_CORPUS = 'PREFIXTEXTLONG' +TARGET = 10 + + +class FakeTokenizer: + """One token per character; fully reversible so token counts are exact.""" + + def encode(self, text, add_special_tokens=False): + return [ord(c) for c in text] + + def decode(self, ids, skip_special_tokens=True): + return ''.join(chr(i) for i in ids) + + def __len__(self): + return 256 + + @property + def all_special_ids(self): + return [] + + +class DriftingTokenizer(FakeTokenizer): + """Encodes ``~`` to a token that ``decode`` drops. + + This reproduces the real round-trip drift cause: prefix fitting decodes with + ``skip_special_tokens=True``, so a naive slice comes back short and has to be + topped up before it matches the budget. + """ + + DROPPED = 0 + + def encode(self, text, add_special_tokens=False): + return [self.DROPPED if c == '~' else ord(c) for c in text] + + def decode(self, ids, skip_special_tokens=True): + return ''.join(chr(i) for i in ids if not (skip_special_tokens and i == self.DROPPED)) + + +def _tok_len(text: str, tokenizer=None) -> int: + return len((tokenizer or FakeTokenizer()).encode(text)) + + +def _write_prefix(tmp_path, text: str) -> str: + path = tmp_path / 'prefix.txt' + path.write_text(text, encoding='utf-8') + return str(path) + + +def _build_line_plugin(tmp_path, monkeypatch, lines, tokenizer=None, apply_chat_template=True, **dataset_args): + monkeypatch.setattr(base_mod, 'load_tokenizer', lambda path: tokenizer or FakeTokenizer()) + path = tmp_path / 'lines.txt' + path.write_text('\n'.join(lines), encoding='utf-8') + url_suffix = 'chat/completions' if apply_chat_template else 'completions' + args = Arguments( + model='test-model', + url=f'http://localhost:8080/v1/{url_suffix}', + dataset='line_by_line', + dataset_path=str(path), + tokenizer_path='fake', + apply_chat_template=apply_chat_template, + dataset_args=dataset_args or None, + ) + return LineByLineDatasetPlugin(args) + + +class TestSingleMessageInjection: + """line_by_line: one user message per request.""" + + def test_system_role_prepends_message(self, tmp_path, monkeypatch): + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abc'], + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + ) + (messages, ) = list(plugin.build_messages()) + assert [m['role'] for m in messages] == ['system', 'user'] + assert messages[1]['content'] == 'abc' + assert sum(_tok_len(m['content']) for m in messages) == TARGET + + def test_user_role_prepends_inline(self, tmp_path, monkeypatch): + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abc'], + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + prefix_role='user', + ) + (messages, ) = list(plugin.build_messages()) + assert [m['role'] for m in messages] == ['user'] + assert messages[0]['content'].endswith('abc') + assert _tok_len(messages[0]['content']) == TARGET + + def test_plain_text_fallback_without_chat_template(self, tmp_path, monkeypatch): + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abc'], + apply_chat_template=False, + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + ) + (prompt, ) = list(plugin.build_messages()) + assert isinstance(prompt, str) + assert prompt.endswith('abc') + assert _tok_len(prompt) == TARGET + + def test_short_corpus_is_tiled(self, tmp_path, monkeypatch): + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abc'], + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, 'PQR'), + ) + (messages, ) = list(plugin.build_messages()) + assert messages[0]['content'] == 'PQRPQRP' + assert sum(_tok_len(m['content']) for m in messages) == TARGET + + def test_tokenizer_round_trip_drift_is_compensated(self, tmp_path, monkeypatch): + tokenizer = DriftingTokenizer() + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abc'], + tokenizer=tokenizer, + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, 'PQ~RS'), + ) + (messages, ) = list(plugin.build_messages()) + prefix = messages[0]['content'] + # The dropped token is gone from the decoded prefix, yet the budget is + # still met exactly because the shortfall was topped up. + assert '~' not in prefix + assert sum(_tok_len(m['content'], tokenizer) for m in messages) == TARGET + + +class TestConversationInjection: + """share_gpt: multi-message conversation, budget measured over all turns.""" + + def _build_plugin(self, tmp_path, monkeypatch, conversation=None, **dataset_args): + monkeypatch.setattr(base_mod, 'load_tokenizer', lambda path: FakeTokenizer()) + conversation = conversation or [{'human': 'hi', 'assistant': 'yo'}, {'human': 'abc', 'assistant': ''}] + path = tmp_path / 'sharegpt.jsonl' + path.write_text(json.dumps({'conversation': conversation}) + '\n', encoding='utf-8') + args = Arguments( + model='test-model', + url='http://localhost:8080/v1/chat/completions', + dataset='share_gpt_zh', + dataset_path=str(path), + tokenizer_path='fake', + apply_chat_template=True, + dataset_args=dataset_args or None, + ) + return ShareGPTZhDatasetPlugin(args) + + def test_system_prefix_fills_whole_conversation_budget(self, tmp_path, monkeypatch): + plugin = self._build_plugin( + tmp_path, + monkeypatch, + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + ) + (messages, ) = list(plugin.build_messages()) + assert [m['role'] for m in messages] == ['system', 'user', 'assistant', 'user'] + # History counts towards the budget: 'hi' + 'yo' + 'abc' = 7 tokens. + assert _tok_len(messages[0]['content']) == TARGET - 7 + assert sum(_tok_len(m['content']) for m in messages) == TARGET + assert messages[-1]['content'] == 'abc' + + def test_user_prefix_goes_to_first_message(self, tmp_path, monkeypatch): + plugin = self._build_plugin( + tmp_path, + monkeypatch, + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + prefix_role='user', + ) + (messages, ) = list(plugin.build_messages()) + assert [m['role'] for m in messages] == ['user', 'assistant', 'user'] + # 3 tokens of budget are left after the 7 tokens of conversation content. + assert messages[0]['content'] == 'PREhi' + assert sum(_tok_len(m['content']) for m in messages) == TARGET + assert messages[-1]['content'] == 'abc' + + def test_conversation_longer_than_target_is_dropped(self, tmp_path, monkeypatch): + # 'hi' + 'yo' + 12 chars = 16 tokens > TARGET; truncating a turn would + # silently change the dialogue, so the whole record is skipped. + plugin = self._build_plugin( + tmp_path, + monkeypatch, + conversation=[{'human': 'hi', 'assistant': 'yo'}, {'human': 'abcdefghijkl', 'assistant': ''}], + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + ) + assert list(plugin.build_messages()) == [] + + def test_no_prefix_fits_last_turn_only(self, tmp_path, monkeypatch): + # Without a prefix_file the legacy path applies: only the last user turn + # is capped, the history is neither counted nor dropped. + plugin = self._build_plugin( + tmp_path, + monkeypatch, + conversation=[{'human': 'a' * 8, 'assistant': 'b' * 8}, {'human': 'abcdefghijkl', 'assistant': ''}], + target_input_len=TARGET, + ) + (messages, ) = list(plugin.build_messages()) + assert messages[0]['content'] == 'a' * 8 # history untouched, not counted + assert messages[-1]['content'] == 'abcdefghij' # last turn capped to TARGET + + def test_no_prefix_drop_filters_last_turn(self, tmp_path, monkeypatch): + # drop is allowed without a prefix and, per legacy behavior, skips a + # conversation whose last turn is shorter than the target. + plugin = self._build_plugin( + tmp_path, + monkeypatch, + conversation=[{'human': 'hi', 'assistant': 'yo'}, {'human': 'abc', 'assistant': ''}], + target_input_len=TARGET, + input_len_mode='drop', + ) + assert list(plugin.build_messages()) == [] + + +class TestNoInjection: + + def test_prompt_filling_the_target_gets_no_prefix(self, tmp_path, monkeypatch): + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abcdefghijkl'], # 12 tokens > target, leaving no prefix budget + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + ) + (messages, ) = list(plugin.build_messages()) + assert [m['role'] for m in messages] == ['user'] + assert _tok_len(messages[0]['content']) == TARGET + + def test_behavior_unchanged_without_prefix_file(self, tmp_path, monkeypatch): + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abcdefghij', 'abc'], + target_input_len=5, + ) + out = list(plugin.build_messages()) + assert out == [[{'role': 'user', 'content': 'abcde'}], [{'role': 'user', 'content': 'abc'}]] + + +class TestLineByLineJsonLines: + """JSON lines bypass prepare_messages, so length control must fail fast.""" + + MESSAGES_LINE = '[{"role": "user", "content": "abc"}]' + BODY_LINE = '{"messages": [{"role": "user", "content": "abc"}], "max_tokens": 8}' + + @pytest.mark.parametrize('json_line', [MESSAGES_LINE, BODY_LINE], ids=['messages_array', 'request_body']) + def test_json_line_rejected_with_length_control(self, tmp_path, monkeypatch, json_line): + plugin = _build_line_plugin( + tmp_path, + monkeypatch, + lines=['abc', json_line], + target_input_len=TARGET, + prefix_file=_write_prefix(tmp_path, PREFIX_CORPUS), + ) + with pytest.raises(ValueError, match='only support plain-text lines'): + list(plugin.build_messages()) + + def test_json_line_rejected_with_target_input_len_only(self, tmp_path, monkeypatch): + plugin = _build_line_plugin(tmp_path, monkeypatch, lines=[self.MESSAGES_LINE], target_input_len=TARGET) + with pytest.raises(ValueError, match='only support plain-text lines'): + list(plugin.build_messages()) + + def test_json_line_passthrough_without_length_control(self, tmp_path, monkeypatch): + plugin = _build_line_plugin(tmp_path, monkeypatch, lines=[self.MESSAGES_LINE, self.BODY_LINE]) + assert list(plugin.build_messages()) == [json.loads(self.MESSAGES_LINE), json.loads(self.BODY_LINE)] + + +@pytest.mark.parametrize( + ('make_args', 'expected'), + [ + (lambda p: {'target_input_len': TARGET, 'prefix_file': str(p / 'nope.txt')}, FileNotFoundError), + (lambda p: {'target_input_len': TARGET, 'prefix_file': _write_prefix(p, '')}, ValueError), + (lambda p: {'prefix_file': _write_prefix(p, PREFIX_CORPUS)}, ValidationError), + ( + lambda p: { + 'target_input_len': TARGET, + 'prefix_file': _write_prefix(p, PREFIX_CORPUS), + 'input_len_mode': 'drop', + }, + ValidationError, + ), + ], + ids=['missing_file', 'empty_file', 'without_target_input_len', 'drop_mode'], +) +def test_invalid_prefix_config_is_rejected(tmp_path, monkeypatch, make_args, expected): + with pytest.raises(expected): + _build_line_plugin(tmp_path, monkeypatch, lines=['abc'], **make_args(tmp_path))