diff --git a/docs/ascend_tutorial/dev_guide/model_dev/parameter_and_metrics.md b/docs/ascend_tutorial/dev_guide/model_dev/parameter_and_metrics.md index 88254f0696b..0aa899b610f 100644 --- a/docs/ascend_tutorial/dev_guide/model_dev/parameter_and_metrics.md +++ b/docs/ascend_tutorial/dev_guide/model_dev/parameter_and_metrics.md @@ -598,7 +598,7 @@ verl 通过层级化的 YAML 配置文件管理所有参数,涉及到的所有 | 参数名 | 默认值 | 说明 | |--------|--------|------| -| `router_replay.mode` | `disabled` | 路由重放模式,可选 disabled、record、replay | +| `actor.megatron.router_replay.mode` / `actor.veomni.router_replay.mode` | `disabled` | 引擎侧路由重放模式,可选 disabled、R2、R3 | | `router_replay.record_file` | `null` | 路由记录文件路径 | | `router_replay.replay_file` | `null` | 路由重放文件路径 | diff --git a/docs/ascend_tutorial/dev_guide/model_dev/transfer_to_npu_guide.md b/docs/ascend_tutorial/dev_guide/model_dev/transfer_to_npu_guide.md index 6a80363c766..b7ad6d3cb29 100644 --- a/docs/ascend_tutorial/dev_guide/model_dev/transfer_to_npu_guide.md +++ b/docs/ascend_tutorial/dev_guide/model_dev/transfer_to_npu_guide.md @@ -155,7 +155,7 @@ class DSAIndexer(MegatronModule): 为了解决这一通用问题,业界引入了 **Routing Replay(路由回放)** 机制。其核心思想是通过锁定特定阶段的专家路由路径,屏蔽微小扰动对路由决策的干扰,从而保证模型训练的稳定性。目前主流包含R2和R3两种变体: -* **(1)Vanilla Routing Replay (R2)**: (对应`actor_rollout_ref.actor.router_replay.mode="R2"`) +* **(1)Vanilla Routing Replay (R2)**: (对应`actor_rollout_ref.actor.megatron.router_replay.mode="R2"`,VeOmni 则为 `actor_rollout_ref.actor.veomni.router_replay.mode="R2"`) * **机制**:在梯度更新阶段,复现训练引擎在上一轮采样阶段计算出的专家路径。 * **作用**:主要缓解**策略陈旧性**对路由的影响。随着策略的更新,当前前向传播计算出的路由可能与生成旧数据时的路由不一致,R2通过回放旧路由来维持优化信号的连贯性。 @@ -169,10 +169,12 @@ class DSAIndexer(MegatronModule): 因此对于大尺寸 MoE 模型,在实际配置中通常推荐使用对齐更彻底的 R3 模式: ``` -actor_rollout_ref.actor.router_replay.mode="R3" \ +actor_rollout_ref.actor.megatron.router_replay.mode="R3" \ actor_rollout_ref.rollout.enable_rollout_routing_replay=True \ ``` +VeOmni 后端请改用 `actor_rollout_ref.actor.veomni.router_replay.mode="R3"`。顶层 `actor.router_replay` 已移除,不会再生效。 + ## 四、性能优化 在昇腾 NPU 上进行大模型 RL(强化学习)训练性能优化时,基础配置调优可优先参考官方文档:[perf_tuning.rst](https://github.com/verl-project/verl/blob/04833f01/docs/perf/perf_tuning.rst)。为实现更高效的优化,建议遵循**数据采集​​→​瓶颈定位​→配置调优→迭代验证**的标准化流程,该流程可显著提升 Rollout、Reward、Update 等核心阶段的吞吐量,同时有效降低资源空转与负载不均问题。性能分析与调优的具体操作,可严格参照以下官方指引: diff --git a/tests/workers/config/test_actor_router_replay_sync_on_cpu.py b/tests/workers/config/test_actor_router_replay_sync_on_cpu.py new file mode 100644 index 00000000000..b1ad65a7a3c --- /dev/null +++ b/tests/workers/config/test_actor_router_replay_sync_on_cpu.py @@ -0,0 +1,46 @@ +# Copyright 2026 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import fields + +from verl.workers.config.actor import ActorConfig, McoreActorConfig, VeOmniActorConfig +from verl.workers.config.engine import EngineRouterReplayConfig, McoreEngineConfig, VeOmniEngineConfig +from verl.workers.config.optimizer import OptimizerConfig + + +def test_actor_config_has_no_top_level_router_replay(): + assert "router_replay" not in {f.name for f in fields(ActorConfig)} + + +def test_mcore_router_replay_lives_on_engine(): + cfg = McoreActorConfig( + rollout_n=1, + ppo_micro_batch_size_per_gpu=1, + megatron=McoreEngineConfig(router_replay=EngineRouterReplayConfig(mode="R3")), + optim=OptimizerConfig(lr=1e-6), + ) + assert not hasattr(cfg, "router_replay") + assert cfg.megatron.router_replay.mode == "R3" + + +def test_veomni_router_replay_lives_on_engine(): + cfg = VeOmniActorConfig( + rollout_n=1, + ppo_micro_batch_size_per_gpu=1, + use_remove_padding=True, + veomni=VeOmniEngineConfig(router_replay=EngineRouterReplayConfig(mode="R2")), + optim=OptimizerConfig(lr=1e-6), + ) + assert not hasattr(cfg, "router_replay") + assert cfg.veomni.router_replay.mode == "R2" diff --git a/verl/experimental/separation/ray_trainer.py b/verl/experimental/separation/ray_trainer.py index 42b6e8aae5d..a7695ba4d9a 100644 --- a/verl/experimental/separation/ray_trainer.py +++ b/verl/experimental/separation/ray_trainer.py @@ -535,7 +535,14 @@ def _fit_compute_log_prob(self, batch: DataProto) -> DataProto: metrics.update(old_log_prob_metrics) old_log_prob.batch.pop("entropys") if "routed_experts" in batch.batch and "routed_experts" in old_log_prob.batch: - router_mode = getattr(self.config.actor_rollout_ref.actor.router_replay, "mode", "disabled") + actor_cfg = self.config.actor_rollout_ref.actor + if getattr(actor_cfg, "strategy", None) == "megatron": + engine_cfg = getattr(actor_cfg, "megatron", None) + elif getattr(actor_cfg, "strategy", None) == "veomni": + engine_cfg = getattr(actor_cfg, "veomni", None) + else: + engine_cfg = None + router_mode = getattr(getattr(engine_cfg, "router_replay", None), "mode", "disabled") if router_mode == "R2": batch.batch.pop("routed_experts") else: diff --git a/verl/trainer/config/_generated_ppo_megatron_trainer.yaml b/verl/trainer/config/_generated_ppo_megatron_trainer.yaml index f8228734e72..0b9bab7ca45 100644 --- a/verl/trainer/config/_generated_ppo_megatron_trainer.yaml +++ b/verl/trainer/config/_generated_ppo_megatron_trainer.yaml @@ -184,11 +184,6 @@ actor_rollout_ref: steps: null stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null} strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False} - router_replay: - _target_: verl.workers.config.RouterReplayConfig - mode: disabled - record_file: null - replay_file: null qat: enable: false mode: w4a16 diff --git a/verl/trainer/config/_generated_ppo_torchtitan_trainer.yaml b/verl/trainer/config/_generated_ppo_torchtitan_trainer.yaml index 0247606a4d8..27fe6676b31 100644 --- a/verl/trainer/config/_generated_ppo_torchtitan_trainer.yaml +++ b/verl/trainer/config/_generated_ppo_torchtitan_trainer.yaml @@ -135,11 +135,6 @@ actor_rollout_ref: steps: null stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null} strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False} - router_replay: - _target_: verl.workers.config.RouterReplayConfig - mode: disabled - record_file: null - replay_file: null qat: enable: false mode: w4a16 diff --git a/verl/trainer/config/_generated_ppo_trainer.yaml b/verl/trainer/config/_generated_ppo_trainer.yaml index d6b3fb72dde..e7af02f9289 100644 --- a/verl/trainer/config/_generated_ppo_trainer.yaml +++ b/verl/trainer/config/_generated_ppo_trainer.yaml @@ -158,11 +158,6 @@ actor_rollout_ref: steps: null stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null} strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False} - router_replay: - _target_: verl.workers.config.RouterReplayConfig - mode: disabled - record_file: null - replay_file: null qat: enable: false mode: w4a16 diff --git a/verl/trainer/config/_generated_ppo_veomni_trainer.yaml b/verl/trainer/config/_generated_ppo_veomni_trainer.yaml index f2a8f11dd64..471a49720ad 100644 --- a/verl/trainer/config/_generated_ppo_veomni_trainer.yaml +++ b/verl/trainer/config/_generated_ppo_veomni_trainer.yaml @@ -150,11 +150,6 @@ actor_rollout_ref: steps: null stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null} strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False} - router_replay: - _target_: verl.workers.config.RouterReplayConfig - mode: disabled - record_file: null - replay_file: null qat: enable: false mode: w4a16 diff --git a/verl/trainer/config/actor/actor.yaml b/verl/trainer/config/actor/actor.yaml index 1a4eb12c647..7c056697983 100644 --- a/verl/trainer/config/actor/actor.yaml +++ b/verl/trainer/config/actor/actor.yaml @@ -286,25 +286,6 @@ profiler: # Whether to fail on unknown stage or missing msprobe strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False} -# Router replay configuration for MoE models -router_replay: - - # Target dataclass for this configuration - _target_: verl.workers.config.RouterReplayConfig - - # Router replay mode: disabled, R2, R3 - # - R2: Use R2 routing strategy (record mode) - # - R3: Use R3 routing strategy (record mode) - mode: disabled - - # File path to save recorded routing decisions - # Required when mode is 'record', 'R2', or 'R3' - record_file: null - - # File path to load recorded routing decisions for replay - # Required when mode is 'replay' - replay_file: null - # QAT (Quantization-Aware Training) configuration # When enabled: # - QAT is automatically applied to actor model during training diff --git a/verl/workers/config/actor.py b/verl/workers/config/actor.py index 33328a24e20..5ca39ca5f18 100644 --- a/verl/workers/config/actor.py +++ b/verl/workers/config/actor.py @@ -136,7 +136,6 @@ class ActorConfig(BaseConfig): optim (OptimizerConfig): Configuration for optimizer. use_fused_kernels (bool): Whether to use custom fused kernels (e.g., FlashAttention, fused MLP). data_loader_seed (int): Seed for data loader. If None, uses global seed. - router_replay (RouterReplayConfig): Configuration for router replay in MoE models. """ _mutable_fields = BaseConfig._mutable_fields | { @@ -185,7 +184,6 @@ class ActorConfig(BaseConfig): engine: BaseConfig = field(default_factory=BaseConfig) rollout_n: int = MISSING # must be override by sampling config model_config: HFModelConfig = field(default_factory=BaseConfig) - router_replay: RouterReplayConfig = field(default_factory=RouterReplayConfig) # Store global batch info for loss aggregation: # dp_size: data parallel size