[cfg, megatron, doc] fix: drop unused actor.router_replay in favor of engine config

docs/ascend_tutorial/dev_guide/model_dev/parameter_and_metrics.md

@@ -598,7 +598,8 @@ verl 通过层级化的 YAML 配置文件管理所有参数，涉及到的所有
 
 | 参数名 | 默认值 | 说明 |
 |--------|--------|------|
-| `router_replay.mode` | `disabled` | 路由重放模式，可选 disabled、record、replay |
+| `actor.megatron.router_replay.mode` / `actor.veomni.router_replay.mode` | `disabled` | 引擎侧路由重放模式，可选 disabled、R2、R3 |
+| `actor.router_replay.mode` | `disabled` | 文档兼容键。非 disabled 且引擎侧仍为 disabled 时复制到引擎配置 |
 | `router_replay.record_file` | `null` | 路由记录文件路径 |
 | `router_replay.replay_file` | `null` | 路由重放文件路径 |
 

docs/ascend_tutorial/dev_guide/model_dev/transfer_to_npu_guide.md

@@ -155,7 +155,7 @@ class DSAIndexer(MegatronModule):
 
 为了解决这一通用问题，业界引入了 **Routing Replay（路由回放）** 机制。其核心思想是通过锁定特定阶段的专家路由路径，屏蔽微小扰动对路由决策的干扰，从而保证模型训练的稳定性。目前主流包含R2和R3两种变体：
 
-* **（1）Vanilla Routing Replay (R2)**： (对应`actor_rollout_ref.actor.router_replay.mode="R2"`)
+* **（1）Vanilla Routing Replay (R2)**： (对应`actor_rollout_ref.actor.megatron.router_replay.mode="R2"`，或顶层 `actor_rollout_ref.actor.router_replay.mode="R2"`。顶层键在引擎侧仍为 `disabled` 时会复制到引擎配置)
 
   * **机制**：在梯度更新阶段，复现训练引擎在上一轮采样阶段计算出的专家路径。
   * **作用**：主要缓解**策略陈旧性**对路由的影响。随着策略的更新，当前前向传播计算出的路由可能与生成旧数据时的路由不一致，R2通过回放旧路由来维持优化信号的连贯性。
@@ -169,10 +169,12 @@ class DSAIndexer(MegatronModule):
 因此对于大尺寸 MoE 模型，在实际配置中通常推荐使用对齐更彻底的 R3 模式：
 
 ```
-actor_rollout_ref.actor.router_replay.mode="R3" \
+actor_rollout_ref.actor.megatron.router_replay.mode="R3" \
 actor_rollout_ref.rollout.enable_rollout_routing_replay=True \
 ```
 
+顶层 `actor_rollout_ref.actor.router_replay.mode="R3"` 同样有效：当 `actor.megatron.router_replay.mode`（或 VeOmni 的对应字段）仍为 `disabled` 时，会自动复制到引擎配置。
+
 ## 四、性能优化
 
 在昇腾 NPU 上进行大模型 RL（强化学习）训练性能优化时，基础配置调优可优先参考官方文档：[perf_tuning.rst](https://github.com/verl-project/verl/blob/04833f01/docs/perf/perf_tuning.rst)。为实现更高效的优化，建议遵循**数据采集​​→​瓶颈定位​→配置调优→迭代验证**的标准化流程，该流程可显著提升 Rollout、Reward、Update 等核心阶段的吞吐量，同时有效降低资源空转与负载不均问题。性能分析与调优的具体操作，可严格参照以下官方指引：

tests/workers/config/test_actor_router_replay_sync_on_cpu.py

@@ -0,0 +1,65 @@
+# Copyright 2026 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from verl.workers.config.actor import McoreActorConfig, RouterReplayConfig, VeOmniActorConfig
+from verl.workers.config.engine import EngineRouterReplayConfig, McoreEngineConfig, VeOmniEngineConfig
+from verl.workers.config.optimizer import OptimizerConfig
+
+
+def test_mcore_copies_actor_router_replay_onto_disabled_engine():
+    cfg = McoreActorConfig(
+        rollout_n=1,
+        ppo_micro_batch_size_per_gpu=1,
+        router_replay=RouterReplayConfig(mode="R3", record_file="rec.json"),
+        megatron=McoreEngineConfig(router_replay=EngineRouterReplayConfig(mode="disabled")),
+        optim=OptimizerConfig(lr=1e-6),
+    )
+    assert cfg.megatron.router_replay.mode == "R3"
+    assert cfg.megatron.router_replay.record_file == "rec.json"
+
+
+def test_veomni_copies_actor_router_replay_onto_disabled_engine():
+    cfg = VeOmniActorConfig(
+        rollout_n=1,
+        ppo_micro_batch_size_per_gpu=1,
+        use_remove_padding=True,
+        router_replay=RouterReplayConfig(mode="R2"),
+        veomni=VeOmniEngineConfig(router_replay=EngineRouterReplayConfig(mode="disabled")),
+        optim=OptimizerConfig(lr=1e-6),
+    )
+    assert cfg.veomni.router_replay.mode == "R2"
+
+
+def test_conflicting_router_replay_modes_raise():
+    with pytest.raises(ValueError, match="Conflicting router_replay modes"):
+        McoreActorConfig(
+            rollout_n=1,
+            ppo_micro_batch_size_per_gpu=1,
+            router_replay=RouterReplayConfig(mode="R2"),
+            megatron=McoreEngineConfig(router_replay=EngineRouterReplayConfig(mode="R3")),
+            optim=OptimizerConfig(lr=1e-6),
+        )
+
+
+def test_engine_mode_kept_when_actor_is_disabled():
+    cfg = McoreActorConfig(
+        rollout_n=1,
+        ppo_micro_batch_size_per_gpu=1,
+        router_replay=RouterReplayConfig(mode="disabled"),
+        megatron=McoreEngineConfig(router_replay=EngineRouterReplayConfig(mode="R3")),
+        optim=OptimizerConfig(lr=1e-6),
+    )
+    assert cfg.megatron.router_replay.mode == "R3"

verl/experimental/separation/ray_trainer.py

@@ -535,7 +535,16 @@ def _fit_compute_log_prob(self, batch: DataProto) -> DataProto:
                 metrics.update(old_log_prob_metrics)
                 old_log_prob.batch.pop("entropys")
                 if "routed_experts" in batch.batch and "routed_experts" in old_log_prob.batch:
-                    router_mode = getattr(self.config.actor_rollout_ref.actor.router_replay, "mode", "disabled")
+                    actor_cfg = self.config.actor_rollout_ref.actor
+                    if getattr(actor_cfg, "strategy", None) == "megatron":
+                        engine_rr = getattr(actor_cfg, "megatron", None)
+                    elif getattr(actor_cfg, "strategy", None) == "veomni":
+                        engine_rr = getattr(actor_cfg, "veomni", None)
+                    else:
+                        engine_rr = None
+                    engine_mode = getattr(getattr(engine_rr, "router_replay", None), "mode", None)
+                    actor_mode = getattr(getattr(actor_cfg, "router_replay", None), "mode", "disabled")
+                    router_mode = engine_mode if engine_mode and engine_mode != "disabled" else actor_mode
                     if router_mode == "R2":
                         batch.batch.pop("routed_experts")
                     else:

verl/trainer/config/actor/actor.yaml

@@ -286,7 +286,9 @@ profiler:
       # Whether to fail on unknown stage or missing msprobe
       strict: ${oc.select:global_profiler.global_tool_config.precision_debugger.strict,False}
 
-# Router replay configuration for MoE models
+# Router replay configuration for MoE models.
+# The Megatron/VeOmni workers read actor.{megatron,veomni}.router_replay.
+# A non-disabled value here is copied onto that engine field when the engine is still disabled.
 router_replay:
 
   # Target dataclass for this configuration

verl/workers/config/actor.py

@@ -221,6 +221,31 @@ def __post_init__(self):
         if self.loss_agg_mode not in valid_loss_agg_modes:
             raise ValueError(f"Invalid loss_agg_mode: {self.loss_agg_mode}")
 
+    def sync_router_replay_to_engine(self, engine) -> None:
+        """Forward ``actor.router_replay`` onto the engine config the worker reads.
+
+        The documented key is ``actor_rollout_ref.actor.router_replay``. The
+        Megatron/VeOmni workers read ``actor.{megatron,veomni}.router_replay``.
+        Copy the actor-level setting when the engine is still disabled, and
+        fail if both are set to different non-disabled modes.
+        """
+        actor_rr = getattr(self, "router_replay", None)
+        engine_rr = getattr(engine, "router_replay", None)
+        if actor_rr is None or engine_rr is None:
+            return
+        actor_mode = getattr(actor_rr, "mode", "disabled")
+        engine_mode = getattr(engine_rr, "mode", "disabled")
+        if actor_mode != "disabled" and engine_mode != "disabled" and actor_mode != engine_mode:
+            raise ValueError(
+                "Conflicting router_replay modes: "
+                f"actor.router_replay.mode={actor_mode!r} vs engine.router_replay.mode={engine_mode!r}. "
+                "Set only one, or make them match."
+            )
+        if actor_mode != "disabled" and engine_mode == "disabled":
+            object.__setattr__(engine_rr, "mode", actor_mode)
+            object.__setattr__(engine_rr, "record_file", getattr(actor_rr, "record_file", None))
+            object.__setattr__(engine_rr, "replay_file", getattr(actor_rr, "replay_file", None))
+
     def validate(self, n_gpus: int, train_batch_size: int, model_config: dict = None):
         """Validate actor configuration with runtime parameters."""
         if not self.use_dynamic_bsz:
@@ -285,6 +310,7 @@ def __post_init__(self):
         """Validate FSDP actor configuration parameters."""
         super().__post_init__()
         self.engine = self.megatron
+        self.sync_router_replay_to_engine(self.megatron)
 
 
 @dataclass
@@ -367,6 +393,7 @@ def __post_init__(self):
         """Validate VeOmni actor configuration parameters."""
         super().__post_init__()
         self.engine = self.veomni
+        self.sync_router_replay_to_engine(self.veomni)
         if self.veomni.router_replay.mode != "disabled" and not self.use_remove_padding:
             raise RuntimeError(
                 "router_replay requires use_remove_padding=True. In VeOmni engine, "