[cfg, megatron, doc] fix: drop unused actor.router_replay in favor of engine config

docs/ascend_tutorial/dev_guide/model_dev/parameter_and_metrics.md

@@ -598,7 +598,7 @@ verl 通过层级化的 YAML 配置文件管理所有参数，涉及到的所有
 
 | 参数名 | 默认值 | 说明 |
 |--------|--------|------|
-| `router_replay.mode` | `disabled` | 路由重放模式，可选 disabled、record、replay |
+| `actor.megatron.router_replay.mode` / `actor.veomni.router_replay.mode` | `disabled` | 引擎侧路由重放模式，可选 disabled、R2、R3 |
 | `router_replay.record_file` | `null` | 路由记录文件路径 |
 | `router_replay.replay_file` | `null` | 路由重放文件路径 |
 

docs/ascend_tutorial/dev_guide/model_dev/transfer_to_npu_guide.md

@@ -155,7 +155,7 @@ class DSAIndexer(MegatronModule):
 
 为了解决这一通用问题，业界引入了 **Routing Replay（路由回放）** 机制。其核心思想是通过锁定特定阶段的专家路由路径，屏蔽微小扰动对路由决策的干扰，从而保证模型训练的稳定性。目前主流包含R2和R3两种变体：
 
-* **（1）Vanilla Routing Replay (R2)**： (对应`actor_rollout_ref.actor.router_replay.mode="R2"`)
+* **（1）Vanilla Routing Replay (R2)**： (对应`actor_rollout_ref.actor.megatron.router_replay.mode="R2"`，VeOmni 则为 `actor_rollout_ref.actor.veomni.router_replay.mode="R2"`)
 
   * **机制**：在梯度更新阶段，复现训练引擎在上一轮采样阶段计算出的专家路径。
   * **作用**：主要缓解**策略陈旧性**对路由的影响。随着策略的更新，当前前向传播计算出的路由可能与生成旧数据时的路由不一致，R2通过回放旧路由来维持优化信号的连贯性。
@@ -169,10 +169,12 @@ class DSAIndexer(MegatronModule):
 因此对于大尺寸 MoE 模型，在实际配置中通常推荐使用对齐更彻底的 R3 模式：
 
 ```
-actor_rollout_ref.actor.router_replay.mode="R3" \
+actor_rollout_ref.actor.megatron.router_replay.mode="R3" \
 actor_rollout_ref.rollout.enable_rollout_routing_replay=True \
 ```
 
+VeOmni 后端请改用 `actor_rollout_ref.actor.veomni.router_replay.mode="R3"`。顶层 `actor.router_replay` 已移除，不会再生效。
+
 ## 四、性能优化
 
 在昇腾 NPU 上进行大模型 RL（强化学习）训练性能优化时，基础配置调优可优先参考官方文档：[perf_tuning.rst](https://github.com/verl-project/verl/blob/04833f01/docs/perf/perf_tuning.rst)。为实现更高效的优化，建议遵循**数据采集​​→​瓶颈定位​→配置调优→迭代验证**的标准化流程，该流程可显著提升 Rollout、Reward、Update 等核心阶段的吞吐量，同时有效降低资源空转与负载不均问题。性能分析与调优的具体操作，可严格参照以下官方指引：

tests/workers/config/test_actor_router_replay_sync_on_cpu.py

@@ -0,0 +1,46 @@
+# Copyright 2026 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import fields
+
+from verl.workers.config.actor import ActorConfig, McoreActorConfig, VeOmniActorConfig
+from verl.workers.config.engine import EngineRouterReplayConfig, McoreEngineConfig, VeOmniEngineConfig
+from verl.workers.config.optimizer import OptimizerConfig
+
+
+def test_actor_config_has_no_top_level_router_replay():
+    assert "router_replay" not in {f.name for f in fields(ActorConfig)}
+
+
+def test_mcore_router_replay_lives_on_engine():
+    cfg = McoreActorConfig(
+        rollout_n=1,
+        ppo_micro_batch_size_per_gpu=1,
+        megatron=McoreEngineConfig(router_replay=EngineRouterReplayConfig(mode="R3")),
+        optim=OptimizerConfig(lr=1e-6),
+    )
+    assert not hasattr(cfg, "router_replay")
+    assert cfg.megatron.router_replay.mode == "R3"
+
+
+def test_veomni_router_replay_lives_on_engine():
+    cfg = VeOmniActorConfig(
+        rollout_n=1,
+        ppo_micro_batch_size_per_gpu=1,
+        use_remove_padding=True,
+        veomni=VeOmniEngineConfig(router_replay=EngineRouterReplayConfig(mode="R2")),
+        optim=OptimizerConfig(lr=1e-6),
+    )
+    assert not hasattr(cfg, "router_replay")
+    assert cfg.veomni.router_replay.mode == "R2"

verl/experimental/separation/ray_trainer.py

@@ -535,7 +535,14 @@ def _fit_compute_log_prob(self, batch: DataProto) -> DataProto:
                 metrics.update(old_log_prob_metrics)
                 old_log_prob.batch.pop("entropys")
                 if "routed_experts" in batch.batch and "routed_experts" in old_log_prob.batch:
-                    router_mode = getattr(self.config.actor_rollout_ref.actor.router_replay, "mode", "disabled")
+                    actor_cfg = self.config.actor_rollout_ref.actor
+                    if getattr(actor_cfg, "strategy", None) == "megatron":
+                        engine_cfg = getattr(actor_cfg, "megatron", None)
+                    elif getattr(actor_cfg, "strategy", None) == "veomni":
+                        engine_cfg = getattr(actor_cfg, "veomni", None)
+                    else:
+                        engine_cfg = None
+                    router_mode = getattr(getattr(engine_cfg, "router_replay", None), "mode", "disabled")
                     if router_mode == "R2":
                         batch.batch.pop("routed_experts")
                     else:

verl/trainer/config/_generated_ppo_megatron_trainer.yaml

@@ -184,11 +184,6 @@ actor_rollout_ref:
           steps: null
           stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null}
           strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False}
-    router_replay:
-      _target_: verl.workers.config.RouterReplayConfig
-      mode: disabled
-      record_file: null
-      replay_file: null
     qat:
       enable: false
       mode: w4a16

verl/trainer/config/_generated_ppo_torchtitan_trainer.yaml

@@ -135,11 +135,6 @@ actor_rollout_ref:
           steps: null
           stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null}
           strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False}
-    router_replay:
-      _target_: verl.workers.config.RouterReplayConfig
-      mode: disabled
-      record_file: null
-      replay_file: null
     qat:
       enable: false
       mode: w4a16

verl/trainer/config/_generated_ppo_trainer.yaml

@@ -158,11 +158,6 @@ actor_rollout_ref:
           steps: null
           stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null}
           strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False}
-    router_replay:
-      _target_: verl.workers.config.RouterReplayConfig
-      mode: disabled
-      record_file: null
-      replay_file: null
     qat:
       enable: false
       mode: w4a16

verl/trainer/config/_generated_ppo_veomni_trainer.yaml

@@ -150,11 +150,6 @@ actor_rollout_ref:
           steps: null
           stages: ${oc.select:global_profiler.global_tool_config.precision_debugger.stages,null}
           strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False}
-    router_replay:
-      _target_: verl.workers.config.RouterReplayConfig
-      mode: disabled
-      record_file: null
-      replay_file: null
     qat:
       enable: false
       mode: w4a16

verl/trainer/config/actor/actor.yaml

@@ -286,25 +286,6 @@ profiler:
       # Whether to fail on unknown stage or missing msprobe
       strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False}
 
-# Router replay configuration for MoE models
-router_replay:
-
-  # Target dataclass for this configuration
-  _target_: verl.workers.config.RouterReplayConfig
-
-  # Router replay mode: disabled, R2, R3
-  # - R2: Use R2 routing strategy (record mode)
-  # - R3: Use R3 routing strategy (record mode)
-  mode: disabled
-
-  # File path to save recorded routing decisions
-  # Required when mode is 'record', 'R2', or 'R3'
-  record_file: null
-
-  # File path to load recorded routing decisions for replay
-  # Required when mode is 'replay'
-  replay_file: null
-
 # QAT (Quantization-Aware Training) configuration
 # When enabled:
 #   - QAT is automatically applied to actor model during training

verl/workers/config/actor.py

@@ -136,7 +136,6 @@ class ActorConfig(BaseConfig):
         optim (OptimizerConfig): Configuration for optimizer.
         use_fused_kernels (bool): Whether to use custom fused kernels (e.g., FlashAttention, fused MLP).
         data_loader_seed (int): Seed for data loader. If None, uses global seed.
-        router_replay (RouterReplayConfig): Configuration for router replay in MoE models.
     """
 
     _mutable_fields = BaseConfig._mutable_fields | {
@@ -185,7 +184,6 @@ class ActorConfig(BaseConfig):
     engine: BaseConfig = field(default_factory=BaseConfig)
     rollout_n: int = MISSING  # must be override by sampling config
     model_config: HFModelConfig = field(default_factory=BaseConfig)
-    router_replay: RouterReplayConfig = field(default_factory=RouterReplayConfig)
 
     # Store global batch info for loss aggregation:
     # dp_size: data parallel size