diff --git a/areal/api/cli_args.py b/areal/api/cli_args.py index 4fdd4aee6a..b78a9025cd 100644 --- a/areal/api/cli_args.py +++ b/areal/api/cli_args.py @@ -256,6 +256,16 @@ class GenerationHyperparameters: ) }, ) + sampling_seed: int | None = field( + default=None, + metadata={ + "help": "Per-request seed for replayable sampling. On the SGLang backend " + "this is forwarded as sampling_params['sampling_seed'], which SGLang only " + "honors when the server is launched with " + "SGLangConfig.enable_deterministic_inference=True; otherwise it is " + "silently ignored. None (default) sends no seed and changes nothing." + }, + ) # NOTE: to add new parameters, please correctly handle them in the `to_openai_args_dict` method. def new(self, **kwargs): @@ -303,6 +313,7 @@ def to_openai_agents_model_settings_dict( "lora_name", # Not supported by OpenAI "use_beam_search", # Not supported by OpenAI "max_tokens", # deprecated by "completions", not used in "responses", should be `max_new_tokens` in "openai-agents" + "sampling_seed", # SGLang-specific; not an OpenAI-compatible parameter } # Workflow-layer flags, not generation arguments. Exclude silently from @@ -2140,6 +2151,10 @@ class SGLangConfig: enable_memory_saver: bool = False allow_auto_truncate: bool = False attention_backend: str | None = "fa3" + # Required for per-request GenerationHyperparameters.sampling_seed to be honored + # (SGLang gates sampling_seed on this flag). Also enables SGLang's batch-invariant + # kernels, at their documented throughput cost, which is why this is opt-in. + enable_deterministic_inference: bool = False enable_multimodal: bool = False sampling_backend: str | None = None context_length: int | None = 32768 @@ -3364,6 +3379,96 @@ def __post_init__(self): # the engine config. Single source of truth: gconfig.lora_name. if self.rollout.use_lora and not self.rollout.lora_name: self.rollout.lora_name = self.gconfig.lora_name + # vLLM has no sampling_seed support (both VLLMBackend and VLLMBridgeBackend + # raise NotImplementedError on the first request); fail here instead, before + # any server launch or model load wastes time on an unsupported config. + # rollout.backend is the current per-engine field, e.g. "vllm:d2t4" or the + # named form "vllm[name]:d2t4" -- confirmed authoritative via + # RolloutController.__init__ parsing config.backend through + # ModelAllocation.from_str, whose grammar allows that optional "[name]", so + # strip it before comparing rather than only splitting on ":". + # Unlike self.sglang below, self.rollout is never None in practice here (the + # `self.rollout.use_lora` check above already dereferences it unconditionally), + # so this ternary is defense-in-depth, not a live guard. + rollout_backend = self.rollout.backend if self.rollout is not None else None + if ( + isinstance(rollout_backend, str) + and rollout_backend.split(":")[0].split("[")[0] == "vllm" + and ( + self.gconfig.sampling_seed is not None + or self.eval_gconfig.sampling_seed is not None + ) + ): + raise ValueError( + "gconfig.sampling_seed or eval_gconfig.sampling_seed is set, but " + "rollout.backend is vLLM, which does not support sampling_seed." + ) + # NOTE: eval_gconfig is currently only consumed for its n_samples field + # (areal/trainer/rl_trainer.py); no stock eval workflow builds a request from + # eval_gconfig's other fields, so eval_gconfig.sampling_seed does not reach + # SGLang either way. Checked here anyway so this warning stays correct if that + # changes, but do not read "no warning" as "eval_gconfig.sampling_seed works." + # self.sglang is not Optional and the YAML/CLI config loader (OmegaConf + # structured-config validation) rejects `sglang: null`, but direct Python + # construction (PPOConfig(sglang=None, ...), bypassing that loader) is not + # type-checked at runtime, so guard rather than assume non-None here. + sglang_deterministic = ( + self.sglang.enable_deterministic_inference + if self.sglang is not None + else False + ) + if ( + self.gconfig.sampling_seed is not None + or self.eval_gconfig.sampling_seed is not None + ) and not sglang_deterministic: + warnings.warn( + "gconfig.sampling_seed or eval_gconfig.sampling_seed is set but " + "sglang.enable_deterministic_inference is False: SGLang silently " + "ignores per-request sampling_seed unless the server is launched " + "with --enable-deterministic-inference. Rollouts will not be seeded " + "as expected. (Not applicable if you're launching SGLang servers " + "yourself outside this config -- the vLLM case is already rejected " + "above, before this warning can be reached.)", + UserWarning, + stacklevel=2, + ) + # Complementary to the warning above (the two never both fire: that one needs + # the flag off, this one needs it on). With deterministic inference ON but no + # per-request seed and n_samples > 1, SGLang gives every seedless request the + # same default seed (42, see sampling_batch_info.py in sglang); its noise is a + # pure function of (seed, position, vocab-index), so a group's same-prompt + # rollouts collapse to identical completions and zero out GRPO's advantage. + # Raise rather than warn (unlike the seed-ignored case above, which is + # harmless): the combination has no working use, and enable_deterministic_ + # inference is new here and default-off, so failing fast breaks no existing + # config -- the same fail-fast this class already applies to sampling_seed on + # the vLLM backend. Scoped to stochastic sampling: under greedy or + # temperature == 0 the request builders decode with temperature 0.0 (see + # "0.0 if gconfig.greedy else gconfig.temperature" in + # SGLangBackend.build_generation_request), so the group collapses regardless of + # any seed and a per-request seed would not change it. Also scoped to a non-vLLM + # backend: enable_deterministic_inference is an SGLang-only flag, so on a vLLM + # run it is inert and this SGLang-worded error would misdirect debugging. The + # vLLM guard above only fires when a seed is set, so it never covers this + # unset-seed case; scope it out here, reusing the rollout_backend parsed above. + if ( + sglang_deterministic + and self.gconfig.sampling_seed is None + and self.gconfig.n_samples > 1 + and not self.gconfig.greedy + and self.gconfig.temperature > 0 + and not ( + isinstance(rollout_backend, str) + and rollout_backend.split(":")[0].split("[")[0] == "vllm" + ) + ): + raise ValueError( + "sglang.enable_deterministic_inference is True with gconfig.n_samples " + "> 1 but no gconfig.sampling_seed: SGLang gives every seedless request " + "the same default seed, so the group's same-prompt rollouts collapse to " + "identical completions. Set distinct per-rollout seeds, use " + "n_samples=1, or disable enable_deterministic_inference." + ) super().__post_init__() diff --git a/areal/engine/sglang_remote.py b/areal/engine/sglang_remote.py index cd54bc8a27..770739823b 100644 --- a/areal/engine/sglang_remote.py +++ b/areal/engine/sglang_remote.py @@ -76,6 +76,9 @@ def build_generation_request( } if stop: sample_params["stop"] = stop + if gconfig.sampling_seed is not None: + # Gating relationship: see GenerationHyperparameters.sampling_seed docstring. + sample_params["sampling_seed"] = gconfig.sampling_seed payload = { "input_ids": req.input_ids.copy(), diff --git a/areal/engine/vllm_remote.py b/areal/engine/vllm_remote.py index 2d76930a9d..b04f0490df 100644 --- a/areal/engine/vllm_remote.py +++ b/areal/engine/vllm_remote.py @@ -62,6 +62,23 @@ def build_generation_request( gconfig = req.gconfig stop_token_ids = gconfig.stop_token_ids + if gconfig.sampling_seed is not None: + # No native vLLM equivalent is wired here (vLLM's own per-request `seed` + # is stream-based and would misalign under this engine's + # interruption/resumption re-submits). Fail loudly rather than silently + # dropping the seed and producing non-reproducible rollouts the caller + # believes are seeded; mirrors how SGLangBackend rejects its own + # unsupported use_beam_search (areal/engine/sglang_remote.py). + # Note: PPOConfig.__post_init__ validates this combination once at + # config-construction time. A seed set dynamically per-request after + # that (e.g. by a future per-rollout seed-minting workflow) still raises + # here, but on the async rollout path this exception is caught + # generically further up the call stack and surfaces as a rejected + # rollout, not a hard crash. + raise NotImplementedError( + "sampling_seed is not yet supported on the vLLM backend." + ) + # NOTE: vLLM uses flat payload structure, not nested sampling_params payload = { "top_p": gconfig.top_p, diff --git a/areal/v2/inference_service/sglang/bridge.py b/areal/v2/inference_service/sglang/bridge.py index 67b835f732..05f2543c31 100644 --- a/areal/v2/inference_service/sglang/bridge.py +++ b/areal/v2/inference_service/sglang/bridge.py @@ -59,6 +59,9 @@ def build_generation_request( } if gconfig.stop: sampling_params["stop"] = gconfig.stop + if gconfig.sampling_seed is not None: + # Gating relationship: see GenerationHyperparameters.sampling_seed docstring. + sampling_params["sampling_seed"] = gconfig.sampling_seed payload: dict[str, Any] = { "input_ids": list(req.input_ids), diff --git a/areal/v2/inference_service/vllm/bridge.py b/areal/v2/inference_service/vllm/bridge.py index bc81c4fded..038e19fdb2 100644 --- a/areal/v2/inference_service/vllm/bridge.py +++ b/areal/v2/inference_service/vllm/bridge.py @@ -34,6 +34,21 @@ def build_generation_request( """Build a ``/v1/completions`` or ``/v1/chat/completions`` request.""" gconfig = req.gconfig + if gconfig.sampling_seed is not None: + # No native vLLM equivalent is wired here; mirrors + # areal.engine.vllm_remote.VLLMBackend's rejection -- fail loudly rather + # than silently drop the seed and produce non-reproducible rollouts the + # caller believes are seeded. + # Note: PPOConfig.__post_init__ validates this combination once at + # config-construction time. A seed set dynamically per-request after + # that (e.g. by a future per-rollout seed-minting workflow) still raises + # here, but on the async rollout path this exception is caught + # generically further up the call stack and surfaces as a rejected + # rollout, not a hard crash. + raise NotImplementedError( + "sampling_seed is not yet supported on the vLLM backend." + ) + # Compute effective max_new_tokens (cap by remaining context window) max_new_tokens = min( gconfig.max_tokens - len(req.input_ids), diff --git a/docs/en/cli_reference.md b/docs/en/cli_reference.md index 87dd6ad241..bf61feb84c 100644 --- a/docs/en/cli_reference.md +++ b/docs/en/cli_reference.md @@ -521,25 +521,26 @@ Core configuration for model training, including optimization and backend settin Controls text generation behavior for rollout. -| Parameter | Type | Default | Description | -| ----------------------- | ---------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `n_samples` | integer | `1` | Number of sequences to generate per prompt. | -| `max_new_tokens` | integer | `16384` | Maximum number of tokens to generate. | -| `min_new_tokens` | integer | `0` | Minimum number of tokens to generate. | -| `max_tokens` | integer | `32768` | Maximum number of tokens including prompt and generated tokens. | -| `greedy` | boolean | `False` | Whether to use greedy decoding (max probability). | -| `top_p` | float | `1.0` | Nucleus sampling probability threshold (0.0, 1.0\]. | -| `top_k` | integer | `100000000` | Number of highest probability tokens to consider. | -| `temperature` | float | `1.0` | Sampling temperature. Higher values increase diversity. | -| `stop_token_ids` | list of integer | `[]` | Stop generation when encountering these token IDs. | -| `ignore_eos` | boolean | `False` | Do not stop generation when EOS is encountered. | -| `skip_special_tokens` | boolean | `True` | Skip special tokens when decoding/displaying outputs. | -| `stop` | list of string \| None | `None` | One or multiple stop words. Generation will stop if one of these words is sampled. | -| `frequency_penalty` | float | `0.0` | Penalizes tokens based on their frequency in generation so far. Must be between -2 and 2 where negative numbers encourage repetition. | -| `lora_name` | string | `"default_lora"` | Lora name to be used for this generation. | -| `use_beam_search` | boolean | `False` | Enable beam search in the vLLM engine. When enabled, sampling parameters like temperature, top-p, and top-k are auto ignored. | -| `reward_normalization` | boolean | `False` | If True, apply per-prompt reward normalization across the n_samples rollouts of the same prompt inside GroupedRolloutWorkflow. Only affects InteractionWithTokenLogpReward workflows such as SWE agent workflows. Not supported by RolloutControllerV2 yet. | -| `drop_incomplete_group` | boolean | `False` | If True, discard the entire group when any of the n_samples rollouts fails or returns None. prepare_batch will automatically retry with a new prompt. This prevents partial groups from causing reward normalization group misalignment. Not supported by RolloutControllerV2 yet. | +| Parameter | Type | Default | Description | +| ----------------------- | ---------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `n_samples` | integer | `1` | Number of sequences to generate per prompt. | +| `max_new_tokens` | integer | `16384` | Maximum number of tokens to generate. | +| `min_new_tokens` | integer | `0` | Minimum number of tokens to generate. | +| `max_tokens` | integer | `32768` | Maximum number of tokens including prompt and generated tokens. | +| `greedy` | boolean | `False` | Whether to use greedy decoding (max probability). | +| `top_p` | float | `1.0` | Nucleus sampling probability threshold (0.0, 1.0\]. | +| `top_k` | integer | `100000000` | Number of highest probability tokens to consider. | +| `temperature` | float | `1.0` | Sampling temperature. Higher values increase diversity. | +| `stop_token_ids` | list of integer | `[]` | Stop generation when encountering these token IDs. | +| `ignore_eos` | boolean | `False` | Do not stop generation when EOS is encountered. | +| `skip_special_tokens` | boolean | `True` | Skip special tokens when decoding/displaying outputs. | +| `stop` | list of string \| None | `None` | One or multiple stop words. Generation will stop if one of these words is sampled. | +| `frequency_penalty` | float | `0.0` | Penalizes tokens based on their frequency in generation so far. Must be between -2 and 2 where negative numbers encourage repetition. | +| `lora_name` | string | `"default_lora"` | Lora name to be used for this generation. | +| `use_beam_search` | boolean | `False` | Enable beam search in the vLLM engine. When enabled, sampling parameters like temperature, top-p, and top-k are auto ignored. | +| `reward_normalization` | boolean | `False` | If True, apply per-prompt reward normalization across the n_samples rollouts of the same prompt inside GroupedRolloutWorkflow. Only affects InteractionWithTokenLogpReward workflows such as SWE agent workflows. Not supported by RolloutControllerV2 yet. | +| `drop_incomplete_group` | boolean | `False` | If True, discard the entire group when any of the n_samples rollouts fails or returns None. prepare_batch will automatically retry with a new prompt. This prevents partial groups from causing reward normalization group misalignment. Not supported by RolloutControllerV2 yet. | +| `sampling_seed` | integer \| None | `None` | Per-request seed for replayable sampling. On the SGLang backend this is forwarded as sampling_params\['sampling_seed'\], which SGLang only honors when the server is launched with SGLangConfig.enable_deterministic_inference=True; otherwise it is silently ignored. None (default) sends no seed and changes nothing. | (section-inference-engine)= @@ -617,6 +618,7 @@ https://github.com/sgl-project/sglang for detailed documentation. | `enable_memory_saver` | boolean | `False` | - | | `allow_auto_truncate` | boolean | `False` | - | | `attention_backend` | string \| None | `"fa3"` | - | +| `enable_deterministic_inference` | boolean | `False` | - | | `enable_multimodal` | boolean | `False` | - | | `sampling_backend` | string \| None | `None` | - | | `context_length` | integer \| None | `32768` | - | diff --git a/docs/zh/cli_reference.md b/docs/zh/cli_reference.md index cdaff25961..cb15d932b4 100644 --- a/docs/zh/cli_reference.md +++ b/docs/zh/cli_reference.md @@ -519,25 +519,26 @@ Core configuration for model training, including optimization and backend settin Controls text generation behavior for rollout. -| Parameter | Type | Default | Description | -| ----------------------- | ---------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `n_samples` | integer | `1` | Number of sequences to generate per prompt. | -| `max_new_tokens` | integer | `16384` | Maximum number of tokens to generate. | -| `min_new_tokens` | integer | `0` | Minimum number of tokens to generate. | -| `max_tokens` | integer | `32768` | Maximum number of tokens including prompt and generated tokens. | -| `greedy` | boolean | `False` | Whether to use greedy decoding (max probability). | -| `top_p` | float | `1.0` | Nucleus sampling probability threshold (0.0, 1.0\]. | -| `top_k` | integer | `100000000` | Number of highest probability tokens to consider. | -| `temperature` | float | `1.0` | Sampling temperature. Higher values increase diversity. | -| `stop_token_ids` | list of integer | `[]` | Stop generation when encountering these token IDs. | -| `ignore_eos` | boolean | `False` | Do not stop generation when EOS is encountered. | -| `skip_special_tokens` | boolean | `True` | Skip special tokens when decoding/displaying outputs. | -| `stop` | list of string \| None | `None` | One or multiple stop words. Generation will stop if one of these words is sampled. | -| `frequency_penalty` | float | `0.0` | Penalizes tokens based on their frequency in generation so far. Must be between -2 and 2 where negative numbers encourage repetition. | -| `lora_name` | string | `"default_lora"` | Lora name to be used for this generation. | -| `use_beam_search` | boolean | `False` | Enable beam search in the vLLM engine. When enabled, sampling parameters like temperature, top-p, and top-k are auto ignored. | -| `reward_normalization` | boolean | `False` | If True, apply per-prompt reward normalization across the n_samples rollouts of the same prompt inside GroupedRolloutWorkflow. Only affects InteractionWithTokenLogpReward workflows such as SWE agent workflows. Not supported by RolloutControllerV2 yet. | -| `drop_incomplete_group` | boolean | `False` | If True, discard the entire group when any of the n_samples rollouts fails or returns None. prepare_batch will automatically retry with a new prompt. This prevents partial groups from causing reward normalization group misalignment. Not supported by RolloutControllerV2 yet. | +| Parameter | Type | Default | Description | +| ----------------------- | ---------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `n_samples` | integer | `1` | Number of sequences to generate per prompt. | +| `max_new_tokens` | integer | `16384` | Maximum number of tokens to generate. | +| `min_new_tokens` | integer | `0` | Minimum number of tokens to generate. | +| `max_tokens` | integer | `32768` | Maximum number of tokens including prompt and generated tokens. | +| `greedy` | boolean | `False` | Whether to use greedy decoding (max probability). | +| `top_p` | float | `1.0` | Nucleus sampling probability threshold (0.0, 1.0\]. | +| `top_k` | integer | `100000000` | Number of highest probability tokens to consider. | +| `temperature` | float | `1.0` | Sampling temperature. Higher values increase diversity. | +| `stop_token_ids` | list of integer | `[]` | Stop generation when encountering these token IDs. | +| `ignore_eos` | boolean | `False` | Do not stop generation when EOS is encountered. | +| `skip_special_tokens` | boolean | `True` | Skip special tokens when decoding/displaying outputs. | +| `stop` | list of string \| None | `None` | One or multiple stop words. Generation will stop if one of these words is sampled. | +| `frequency_penalty` | float | `0.0` | Penalizes tokens based on their frequency in generation so far. Must be between -2 and 2 where negative numbers encourage repetition. | +| `lora_name` | string | `"default_lora"` | Lora name to be used for this generation. | +| `use_beam_search` | boolean | `False` | Enable beam search in the vLLM engine. When enabled, sampling parameters like temperature, top-p, and top-k are auto ignored. | +| `reward_normalization` | boolean | `False` | If True, apply per-prompt reward normalization across the n_samples rollouts of the same prompt inside GroupedRolloutWorkflow. Only affects InteractionWithTokenLogpReward workflows such as SWE agent workflows. Not supported by RolloutControllerV2 yet. | +| `drop_incomplete_group` | boolean | `False` | If True, discard the entire group when any of the n_samples rollouts fails or returns None. prepare_batch will automatically retry with a new prompt. This prevents partial groups from causing reward normalization group misalignment. Not supported by RolloutControllerV2 yet. | +| `sampling_seed` | integer \| None | `None` | Per-request seed for replayable sampling. On the SGLang backend this is forwarded as sampling_params\['sampling_seed'\], which SGLang only honors when the server is launched with SGLangConfig.enable_deterministic_inference=True; otherwise it is silently ignored. None (default) sends no seed and changes nothing. | (section-inference-engine)= @@ -615,6 +616,7 @@ https://github.com/sgl-project/sglang for detailed documentation. | `enable_memory_saver` | boolean | `False` | - | | `allow_auto_truncate` | boolean | `False` | - | | `attention_backend` | string \| None | `"fa3"` | - | +| `enable_deterministic_inference` | boolean | `False` | - | | `enable_multimodal` | boolean | `False` | - | | `sampling_backend` | string \| None | `None` | - | | `context_length` | integer \| None | `32768` | - | diff --git a/tests/test_sampling_seed_deterministic_inference_warning.py b/tests/test_sampling_seed_deterministic_inference_warning.py new file mode 100644 index 0000000000..1b07e1c80a --- /dev/null +++ b/tests/test_sampling_seed_deterministic_inference_warning.py @@ -0,0 +1,202 @@ +import warnings + +import pytest + +from areal.api.cli_args import ( + GenerationHyperparameters, + InferenceEngineConfig, + PPOConfig, + SGLangConfig, +) + + +def test_ppo_config_warns_when_sampling_seed_set_without_deterministic_inference(): + """SGLang silently ignores per-request sampling_seed unless the server runs with + --enable-deterministic-inference (SGLangConfig.enable_deterministic_inference). + Since both fields live on the same PPOConfig, catch the common misconfiguration + at config-construction time rather than leaving it a silent no-op discoverable + only by reading SGLang internals.""" + with pytest.warns(UserWarning, match="sampling_seed is set but"): + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(sampling_seed=42), + ) + + +def test_ppo_config_does_not_warn_when_flags_are_consistent(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(sampling_seed=42), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + assert not any("sampling_seed is set but" in str(w.message) for w in caught) + + +def test_ppo_config_does_not_warn_when_sampling_seed_unset(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + PPOConfig(experiment_name="exp", trial_name="trial") + + assert not any("sampling_seed is set but" in str(w.message) for w in caught) + + +def test_ppo_config_warns_when_eval_sampling_seed_set_without_deterministic_inference(): + """eval_gconfig can carry its own sampling_seed independent of gconfig (e.g. a + fixed seed for held-out eval while training rollouts are unseeded); the check + must not miss it just because gconfig itself has no seed set.""" + with pytest.warns(UserWarning, match="sampling_seed is set but"): + PPOConfig( + experiment_name="exp", + trial_name="trial", + eval_gconfig=GenerationHyperparameters(sampling_seed=42), + ) + + +def test_ppo_config_does_not_warn_when_eval_flags_are_consistent(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + PPOConfig( + experiment_name="exp", + trial_name="trial", + eval_gconfig=GenerationHyperparameters(sampling_seed=42), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + assert not any("sampling_seed is set but" in str(w.message) for w in caught) + + +def test_ppo_config_warns_without_crashing_when_sglang_is_none(): + """sglang is not Optional and the YAML/CLI loader rejects `sglang: null`, but + direct Python construction (PPOConfig(sglang=None, ...)) bypasses that loader and + is not type-checked at runtime, so this must not raise AttributeError.""" + with pytest.warns(UserWarning, match="sampling_seed is set but"): + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(sampling_seed=42), + sglang=None, + ) + + +def test_ppo_config_raises_when_sampling_seed_set_with_vllm_backend(): + """vLLM has no sampling_seed support; both VLLMBackend and VLLMBridgeBackend + raise NotImplementedError on the first generation request, so fail fast at + config-construction time instead of after server launch and model load.""" + with pytest.raises(ValueError, match="does not support sampling_seed"): + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(sampling_seed=42), + rollout=InferenceEngineConfig(backend="vllm:d2t4"), + ) + + +def test_ppo_config_does_not_raise_when_vllm_backend_without_sampling_seed(): + PPOConfig( + experiment_name="exp", + trial_name="trial", + rollout=InferenceEngineConfig(backend="vllm:d2t4"), + ) + + +def test_ppo_config_does_not_raise_when_sglang_backend_with_sampling_seed(): + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(sampling_seed=42), + rollout=InferenceEngineConfig(backend="sglang:d2t4"), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + +def test_ppo_config_does_not_raise_when_rollout_backend_unset(): + """Default InferenceEngineConfig().backend is the OmegaConf MISSING sentinel + ('???'), not a real backend string; must not crash or misfire as vLLM.""" + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(sampling_seed=42), + ) + + +def test_ppo_config_raises_on_grouped_deterministic_rollouts_without_seed(): + """With deterministic inference on but no per-request seed, SGLang gives every + seedless request the same default seed, so a group's n_samples>1 same-prompt + rollouts collapse to identical completions and zero out GRPO's advantage. This + combination has no working use and the flag is new and default-off, so fail fast + at config time rather than burning a whole training run on a degenerate config.""" + with pytest.raises(ValueError, match="collapse to identical completions"): + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(n_samples=8), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + +def test_ppo_config_does_not_raise_on_grouped_deterministic_rollouts_with_seed(): + """A seed IS set, the user's signal they intend distinct per-rollout seeds, so + the collapse guard must not fire.""" + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(n_samples=8, sampling_seed=42), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + +def test_ppo_config_does_not_raise_on_deterministic_single_sample_without_seed(): + """n_samples == 1 is not a group, so there is no intra-group diversity to + collapse; the guard must not fire (it would otherwise block every non-grouped + run that enables deterministic inference).""" + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(n_samples=1), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + +def test_ppo_config_does_not_raise_on_grouped_deterministic_greedy_without_seed(): + """Under greedy decoding the request builders decode at temperature 0.0, so the + group collapses regardless of any seed and a per-request seed would not change + it; the seed-focused guard is scoped to stochastic sampling and must not fire.""" + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(n_samples=8, greedy=True), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + +def test_ppo_config_does_not_raise_on_grouped_deterministic_temperature_zero(): + """temperature == 0 is likewise seed-independent argmax decoding; the guard is + scoped to temperature > 0 and must not fire.""" + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(n_samples=8, temperature=0.0), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) + + +def test_ppo_config_does_not_raise_on_grouped_deterministic_vllm_backend(): + """enable_deterministic_inference is an SGLang-only flag and inert on vLLM, so + the SGLang-worded collapse guard must not fire for a vLLM backend even when the + flag is (pointlessly) set; a vLLM run does not hit SGLang's default-seed path.""" + PPOConfig( + experiment_name="exp", + trial_name="trial", + gconfig=GenerationHyperparameters(n_samples=8), + rollout=InferenceEngineConfig(backend="vllm:d2t4"), + sglang=SGLangConfig(enable_deterministic_inference=True), + ) diff --git a/tests/test_sglang_bridge_generation_request.py b/tests/test_sglang_bridge_generation_request.py new file mode 100644 index 0000000000..29f00b8987 --- /dev/null +++ b/tests/test_sglang_bridge_generation_request.py @@ -0,0 +1,34 @@ +from areal.api.cli_args import GenerationHyperparameters +from areal.api.io_struct import ModelRequest +from areal.v2.inference_service.sglang.bridge import SGLangBridgeBackend + + +def test_sglang_bridge_forwards_sampling_seed_when_set(): + """SGLangBridgeBackend mirrors SGLangBackend's sampling_seed forwarding (its own + docstring says it "Mirrors the relevant subset of ... SGLangBackend") -- the v2 + data-proxy path must not silently diverge from the v1 remote-engine path.""" + gconfig = GenerationHyperparameters(max_new_tokens=8, sampling_seed=12345) + req = ModelRequest(input_ids=[11, 12], gconfig=gconfig) + + payload = ( + SGLangBridgeBackend() + .build_generation_request(req, with_lora=False, version=0) + .payload + ) + + assert payload["sampling_params"]["sampling_seed"] == 12345 + + +def test_sglang_bridge_omits_sampling_seed_by_default(): + """Default (None) must send no seed field at all, so existing deployments and + requests are byte-for-byte unaffected.""" + gconfig = GenerationHyperparameters(max_new_tokens=8) + req = ModelRequest(input_ids=[11, 12], gconfig=gconfig) + + payload = ( + SGLangBridgeBackend() + .build_generation_request(req, with_lora=False, version=0) + .payload + ) + + assert "sampling_seed" not in payload["sampling_params"] diff --git a/tests/test_sglang_deterministic_inference_flag.py b/tests/test_sglang_deterministic_inference_flag.py new file mode 100644 index 0000000000..9a908288d6 --- /dev/null +++ b/tests/test_sglang_deterministic_inference_flag.py @@ -0,0 +1,36 @@ +from areal.api.cli_args import SGLangConfig + + +def _build_cmd(monkeypatch, **config_kwargs): + # SGLangConfig.build_args guards on the `sglang` package being installed to check + # its version (cli_args.py: `pkg_version.is_version_greater_or_equal("sglang", ...)`). + # `sglang` is a GPU-only Linux package, unavailable on this machine and orthogonal + # to the config-passthrough logic under test, so stub the version check rather than + # bypass build_args/build_cmd_from_args (the real entry points SGLang launches use). + monkeypatch.setattr( + "areal.api.cli_args.pkg_version.is_version_greater_or_equal", + lambda *a, **kw: True, + ) + config = SGLangConfig(model_path="dummy", **config_kwargs) + return SGLangConfig.build_cmd( + sglang_config=config, + tp_size=1, + base_gpu_id=0, + dist_init_addr="127.0.0.1:12345", + ) + + +def test_enable_deterministic_inference_flag_passed_through_when_true(monkeypatch): + """Required for GenerationHyperparameters.sampling_seed to be honored by SGLang + (SGLang gates per-request sampling_seed on this server flag).""" + cmd = _build_cmd(monkeypatch, enable_deterministic_inference=True) + + assert "--enable-deterministic-inference" in cmd + + +def test_enable_deterministic_inference_flag_omitted_by_default(monkeypatch): + """Default (False) must not appear on the command line, so existing SGLang + launches are byte-for-byte unaffected.""" + cmd = _build_cmd(monkeypatch) + + assert "--enable-deterministic-inference" not in cmd diff --git a/tests/test_sglang_generation_request.py b/tests/test_sglang_generation_request.py new file mode 100644 index 0000000000..14de39cadf --- /dev/null +++ b/tests/test_sglang_generation_request.py @@ -0,0 +1,33 @@ +from areal.api.cli_args import GenerationHyperparameters +from areal.api.io_struct import ModelRequest +from areal.engine.sglang_remote import SGLangBackend + + +def test_sglang_forwards_sampling_seed_when_set(): + """When sampling_seed is set, it must reach sample_params so SGLang's seeded + Gumbel sampler (multinomial_with_seed) can consume it.""" + gconfig = GenerationHyperparameters(max_new_tokens=8, sampling_seed=12345) + req = ModelRequest(input_ids=[11, 12], gconfig=gconfig) + + payload = ( + SGLangBackend() + .build_generation_request(req, with_lora=False, version=0) + .payload + ) + + assert payload["sampling_params"]["sampling_seed"] == 12345 + + +def test_sglang_omits_sampling_seed_by_default(): + """Default (None) must send no seed field at all, so existing deployments and + requests are byte-for-byte unaffected.""" + gconfig = GenerationHyperparameters(max_new_tokens=8) + req = ModelRequest(input_ids=[11, 12], gconfig=gconfig) + + payload = ( + SGLangBackend() + .build_generation_request(req, with_lora=False, version=0) + .payload + ) + + assert "sampling_seed" not in payload["sampling_params"] diff --git a/tests/test_vllm_generation_request.py b/tests/test_vllm_generation_request.py index 18a42d0ee8..1f801ba62e 100644 --- a/tests/test_vllm_generation_request.py +++ b/tests/test_vllm_generation_request.py @@ -1,3 +1,5 @@ +import pytest + from areal.api.cli_args import GenerationHyperparameters from areal.api.io_struct import ModelRequest from areal.engine.vllm_remote import VLLMBackend @@ -18,3 +20,15 @@ def test_vllm_forwards_frequency_penalty_and_stop(): assert payload["frequency_penalty"] == 0.5 assert payload["stop"] == ["STOP"] + + +def test_vllm_rejects_sampling_seed(): + """sampling_seed has no vLLM wiring here (unlike SGLang, which forwards it under + enable_deterministic_inference). Must fail loudly -- mirroring how SGLangBackend + already rejects its own unsupported param (use_beam_search) -- rather than + silently no-op and leave a caller believing their rollouts are seeded.""" + gconfig = GenerationHyperparameters(max_new_tokens=8, sampling_seed=12345) + req = ModelRequest(input_ids=[11, 12], gconfig=gconfig) + + with pytest.raises(NotImplementedError): + VLLMBackend().build_generation_request(req, with_lora=False, version=0) diff --git a/tests/v2/inference_service/test_inf_bridge.py b/tests/v2/inference_service/test_inf_bridge.py index aad70d546b..d79475ab18 100644 --- a/tests/v2/inference_service/test_inf_bridge.py +++ b/tests/v2/inference_service/test_inf_bridge.py @@ -69,6 +69,7 @@ def _make_request( temperature: float = 1.0, metadata: dict[str, Any] | None = None, lora_name: str | None = None, + sampling_seed: int | None = None, ) -> ModelRequest: """Create a ModelRequest with sensible defaults for testing.""" if input_ids is None: @@ -79,6 +80,7 @@ def _make_request( max_tokens=max_tokens, greedy=greedy, temperature=temperature, + sampling_seed=sampling_seed, ) if lora_name is not None: gconfig.lora_name = lora_name @@ -473,6 +475,15 @@ def test_vllm_build_generation_request_for_text(self): assert http_req.payload["max_tokens"] == 7 assert http_req.payload["stream"] is False + def test_vllm_bridge_rejects_sampling_seed(self): + """No vLLM wiring for sampling_seed here; must fail loudly rather than + silently no-op, mirroring areal.engine.vllm_remote.VLLMBackend.""" + backend = VLLMBridgeBackend() + req = _make_request(input_ids=[11, 12], sampling_seed=12345) + + with pytest.raises(NotImplementedError): + backend.build_generation_request(req, with_lora=False, version=0) + def test_vllm_parse_generation_response_for_chat_format(self): """vLLM bridge parses chat logprobs content format.""" backend = VLLMBridgeBackend()