diff --git a/tests/experimental/rollout/inprocess_vllm_sampler_adapter_test.py b/tests/experimental/rollout/inprocess_vllm_sampler_adapter_test.py index d50f88560..c44518308 100644 --- a/tests/experimental/rollout/inprocess_vllm_sampler_adapter_test.py +++ b/tests/experimental/rollout/inprocess_vllm_sampler_adapter_test.py @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for InprocessVllmSamplerAdapter with Tunix VllmSampler and Raiden delegate.""" +"""Tests for InprocessVllmSamplerAdapter with Tunix VllmSampler.""" import asyncio from unittest import mock from absl.testing import absltest import numpy as np + from tunix.experimental.rollout import inprocess_vllm_sampler_adapter from tunix.experimental.rollout import sampler as base_sampler_lib -from tunix.experimental.weight_sync import raiden_weight_sync_delegate -from tunix.experimental.weight_sync import weight_sync from tunix.generate import base_sampler @@ -37,7 +36,6 @@ def setUp(self): padded_prompt_tokens=np.array([[1, 2]], dtype=np.int32), logprobs=None, ) - self.mock_vllm_sampler.mesh = "mock_mesh" self.mock_vllm_lib = mock.MagicMock() self.mock_vllm_lib.VllmSampler.return_value = self.mock_vllm_sampler @@ -50,7 +48,6 @@ def setUp(self): self.mock_tokenizer = mock.MagicMock() self.mock_config = mock.MagicMock() - self.mock_config.enable_raiden = False self.sampler_adapter = ( inprocess_vllm_sampler_adapter.InprocessVllmSamplerAdapter( @@ -64,17 +61,6 @@ def tearDown(self): self.patcher.stop() super().tearDown() - def test_implements_sampler_protocol(self): - self.assertIsInstance(self.sampler_adapter, base_sampler_lib.Sampler) - - def test_lifecycle_methods(self): - self.assertTrue(asyncio.run(self.sampler_adapter.start())) - self.assertTrue(asyncio.run(self.sampler_adapter.pause())) - self.assertTrue(asyncio.run(self.sampler_adapter.resume())) - self.assertEqual(asyncio.run(self.sampler_adapter.get_mesh()), "mock_mesh") - self.assertTrue(asyncio.run(self.sampler_adapter.stop())) - self.mock_vllm_sampler.stop.assert_called_once() - def test_single_sampling_request(self): req = base_sampler_lib.SamplingRequest( request_id="vllm_req_01", @@ -121,123 +107,12 @@ def test_batch_sampling_requests(self): np.testing.assert_array_equal(responses[0].prompt_token_ids, [1, 2]) np.testing.assert_array_equal(responses[1].prompt_token_ids, [3, 4]) - def test_weight_sync_without_raiden_delegate(self): + def test_weight_sync(self): mock_weights = {"layer1": "weights"} - req = base_sampler_lib.WeightSyncRequest(weights=mock_weights) - res = asyncio.run(self.sampler_adapter.weight_sync(sync_request=req)) + res = asyncio.run(self.sampler_adapter.weight_sync(mock_weights)) self.assertTrue(res) self.mock_vllm_sampler.update_params.assert_called_once_with(mock_weights) - # Missing sync_request should raise ValueError - with self.assertRaises(ValueError): - asyncio.run(self.sampler_adapter.weight_sync(sync_request=None)) - - # Missing weights in sync_request should raise ValueError - empty_req = base_sampler_lib.WeightSyncRequest() - with self.assertRaises(ValueError): - asyncio.run(self.sampler_adapter.weight_sync(sync_request=empty_req)) - - self.assertIsNone(asyncio.run(self.sampler_adapter.bind_weight_sync())) - self.assertTrue(asyncio.run(self.sampler_adapter.pre_weight_sync())) - self.assertTrue(asyncio.run(self.sampler_adapter.post_weight_sync())) - with self.assertRaises(NotImplementedError): - asyncio.run(self.sampler_adapter.get_weight_sync_metadata()) - - def test_weight_sync_with_raiden_delegate(self): - mock_delegate = mock.MagicMock( - spec=raiden_weight_sync_delegate.RaidenWeightSyncDelegate - ) - mock_delegate.is_bounded.return_value = False - mock_delegate.bind_weight_sync = mock.AsyncMock(return_value=True) - mock_delegate.get_weight_sync_metadata = mock.AsyncMock( - return_value=[{"unit": "rollout"}] - ) - mock_delegate.pre_weight_sync = mock.AsyncMock(return_value=True) - mock_delegate.weight_sync = mock.AsyncMock(return_value=5) - mock_delegate.post_weight_sync = mock.AsyncMock(return_value=True) - - fake_transformer_state = {"param": "tensor"} - self.mock_vllm_sampler.transformer_state = fake_transformer_state - - raiden_config = mock.MagicMock() - raiden_config.weight_sync_mode = weight_sync.WeightSyncMode.RAIDEN - - raiden_adapter = inprocess_vllm_sampler_adapter.InprocessVllmSamplerAdapter( - server_id="vllm_raiden_slice", - tokenizer=self.mock_tokenizer, - config=raiden_config, - raiden_sync_delegate=mock_delegate, - ) - - sync_req = base_sampler_lib.WeightSyncRequest(policy_version=5) - - # 1. bind_weight_sync - asyncio.run(raiden_adapter.bind_weight_sync(sync_req)) - mock_delegate.bind_weight_sync.assert_awaited_once_with( - sync_request=sync_req, state=fake_transformer_state - ) - mock_delegate.is_bounded.return_value = True - - # 2. get_weight_sync_metadata - metadata = asyncio.run(raiden_adapter.get_weight_sync_metadata()) - self.assertEqual(metadata, [{"unit": "rollout"}]) - - # 3. pre_weight_sync - self.assertTrue(asyncio.run(raiden_adapter.pre_weight_sync(sync_req))) - mock_delegate.pre_weight_sync.assert_awaited_once_with( - sync_request=sync_req - ) - - # 4. weight_sync - version = asyncio.run(raiden_adapter.weight_sync(sync_req)) - self.assertEqual(version, 5) - mock_delegate.weight_sync.assert_awaited_once_with(sync_request=sync_req) - - # 5. post_weight_sync - self.assertTrue(asyncio.run(raiden_adapter.post_weight_sync(sync_req))) - mock_delegate.post_weight_sync.assert_awaited_once_with( - sync_request=sync_req - ) - - def test_raiden_bind_without_transformer_state_raises(self): - mock_delegate = mock.MagicMock( - spec=raiden_weight_sync_delegate.RaidenWeightSyncDelegate - ) - mock_delegate.is_bounded.return_value = False - # Ensure vllm_sampler has no transformer_state - if hasattr(self.mock_vllm_sampler, "transformer_state"): - del self.mock_vllm_sampler.transformer_state - - raiden_config = mock.MagicMock() - raiden_config.weight_sync_mode = weight_sync.WeightSyncMode.RAIDEN - - raiden_adapter = inprocess_vllm_sampler_adapter.InprocessVllmSamplerAdapter( - server_id="vllm_raiden_slice", - tokenizer=self.mock_tokenizer, - config=raiden_config, - raiden_sync_delegate=mock_delegate, - ) - - with self.assertRaisesRegex(RuntimeError, "transformer_state"): - asyncio.run(raiden_adapter.bind_weight_sync()) - - def test_other_sampler_methods(self): - self.assertEqual( - asyncio.run(self.sampler_adapter.get_transfer_status("req_1")), - "SUCCESS", - ) - self.assertTrue( - asyncio.run( - self.sampler_adapter.migrate_kv_cache( - source_server_id="s1", - target_server_id="s2", - token_ids=[1, 2, 3], - ) - ) - ) - load_info = asyncio.run(self.sampler_adapter.get_load_info()) - self.assertIsInstance(load_info, base_sampler_lib.LoadInfo) - def test_uninitialized_sampler_raises(self): uninit = inprocess_vllm_sampler_adapter.InprocessVllmSamplerAdapter( server_id="empty" diff --git a/tests/experimental/rollout/manager_test.py b/tests/experimental/rollout/manager_test.py index 921a040e1..18bf3c249 100644 --- a/tests/experimental/rollout/manager_test.py +++ b/tests/experimental/rollout/manager_test.py @@ -13,14 +13,11 @@ # limitations under the License. import asyncio -import types import unittest -from unittest import mock from absl.testing import absltest from tunix.experimental.rollout import manager as manager_lib from tunix.experimental.rollout import sampler as sampler_lib -from tunix.experimental.weight_sync import weight_sync class _FakeSampler(sampler_lib.Sampler): @@ -94,7 +91,7 @@ async def test_post_reopens_admission(self): await manager.pre_weight_sync() await manager.post_weight_sync() self.assertTrue(manager._traffic.is_admission_open()) - + async def test_reopen_admission_after_abort(self): manager = self._manager() await manager.pre_weight_sync() @@ -140,48 +137,5 @@ async def test_drain_timeout_returns(self): task.cancel() manager._active_tasks.pop("t0", None) - -class WeightSyncModeTest(absltest.TestCase): - - def test_config_weight_sync_mode_raiden(self): - config = types.SimpleNamespace( - sampler_type="vanilla", - weight_sync_mode=weight_sync.WeightSyncMode.RAIDEN, - ) - manager = manager_lib.RolloutManager( - config=config, tokenizer="mock", chat_parser="mock" - ) - self.assertTrue(getattr(manager.sampler, "enable_raiden", False)) - self.assertIsNotNone(getattr(manager.sampler, "raiden_sync_delegate", None)) - - def test_config_weight_sync_mode_fallback(self): - config = types.SimpleNamespace( - sampler_type="vanilla", - weight_sync_mode=weight_sync.WeightSyncMode.FALLBACK, - ) - manager = manager_lib.RolloutManager( - config=config, tokenizer="mock", chat_parser="mock" - ) - self.assertFalse(getattr(manager.sampler, "enable_raiden", False)) - self.assertIsNone(getattr(manager.sampler, "raiden_sync_delegate", None)) - - @mock.patch( - "tunix.experimental.rollout.inprocess_vllm_sampler_adapter._get_vllm_sampler_cls" - ) - def test_config_weight_sync_mode_inprocess_vllm_raiden(self, mock_get_vllm): - mock_lib = mock.MagicMock() - mock_lib.VllmSampler.return_value = mock.MagicMock() - mock_get_vllm.return_value = mock_lib - config = types.SimpleNamespace( - sampler_type="inprocess_vllm", - weight_sync_mode=weight_sync.WeightSyncMode.RAIDEN, - ) - manager = manager_lib.RolloutManager( - config=config, tokenizer="mock", chat_parser="mock" - ) - self.assertTrue(getattr(manager.sampler, "enable_raiden", False)) - self.assertIsNotNone(getattr(manager.sampler, "raiden_sync_delegate", None)) - - if __name__ == "__main__": absltest.main() diff --git a/tests/experimental/weight_sync/raiden_weight_sync_delegate_test.py b/tests/experimental/rollout/raiden_sampler_adapter_test.py similarity index 50% rename from tests/experimental/weight_sync/raiden_weight_sync_delegate_test.py rename to tests/experimental/rollout/raiden_sampler_adapter_test.py index 38e7b5554..10459588b 100644 --- a/tests/experimental/weight_sync/raiden_weight_sync_delegate_test.py +++ b/tests/experimental/rollout/raiden_sampler_adapter_test.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests verifying destination-side Raiden weight sync delegation.""" +"""Tests for RaidenSamplerAdapter.""" import unittest from unittest import mock from absl.testing import absltest -from tunix.experimental.weight_sync import raiden_weight_sync_delegate +from tunix.experimental.rollout import raiden_sampler_adapter class _FakeWorker: @@ -30,11 +30,10 @@ def __init__(self, job_name, worker_index=0, **kwargs): self.bound = False self.bound_state = None self.h2d_calls = 0 - self.bind_calls = 0 def bind(self, state): self.bound = True - self.bind_calls += 1 + self.bind_calls = getattr(self, "bind_calls", 0) + 1 self.bound_state = state def work_unit_metadata(self): @@ -50,79 +49,92 @@ def checksums(self): return {} +class _FakeSampler: + transformer_state = {"w": 1} + + class _Request: def __init__(self, policy_version): self.policy_version = policy_version -class RaidenWeightSyncDelegateTest(unittest.IsolatedAsyncioTestCase): +class RaidenSamplerAdapterTest(unittest.IsolatedAsyncioTestCase): def setUp(self): super().setUp() patcher = mock.patch.object( - raiden_weight_sync_delegate.raiden_synchronizer, + raiden_sampler_adapter.raiden_synchronizer, "RaidenSynchronizer", _FakeWorker, ) patcher.start() self.addCleanup(patcher.stop) - def _delegate(self): - return raiden_weight_sync_delegate.RaidenWeightSyncDelegate() + def _adapter(self): + adapter = raiden_sampler_adapter.RaidenSamplerAdapter( + server_id="test_sampler" + ) + adapter.sampler = _FakeSampler() + return adapter - async def test_bind_binds_state(self): - delegate = self._delegate() - fake_state = {"w": 1} - await delegate.bind_weight_sync(state=fake_state) - worker = delegate._synchronizers[0] - self.assertIs(worker.bound_state, fake_state) + async def test_bind_binds_sampler_state(self): + adapter = self._adapter() + await adapter.bind_weight_sync() + worker = adapter._synchronizers[0] + self.assertIs(worker.bound_state, adapter.sampler.transformer_state) async def test_worker_uses_the_validated_config(self): - delegate = self._delegate() - self.assertIs(delegate._synchronizers[0].kwargs["auto_h2d"], True) + adapter = self._adapter() + self.assertIs(adapter._synchronizers[0].kwargs["auto_h2d"], True) async def test_repeat_phases_bind_exactly_once(self): - delegate = self._delegate() - fake_state = {"w": 1} - await delegate.bind_weight_sync(state=fake_state) - await delegate.get_weight_sync_metadata() - await delegate.pre_weight_sync() - await delegate.weight_sync() - self.assertEqual(delegate._synchronizers[0].bind_calls, 1) + adapter = self._adapter() + await adapter.bind_weight_sync() + await adapter.get_weight_sync_metadata() + await adapter.pre_weight_sync() + await adapter.weight_sync() + self.assertEqual(adapter._synchronizers[0].bind_calls, 1) async def test_metadata_returns_one_entry_per_worker(self): - delegate = self._delegate() - md = await delegate.get_weight_sync_metadata() + adapter = self._adapter() + md = await adapter.get_weight_sync_metadata() self.assertEqual(md, [{"unit": "rollout"}]) async def test_weight_sync_installs_and_tracks_version(self): - delegate = self._delegate() - await delegate.bind_weight_sync(state={"w": 1}) - version = await delegate.weight_sync(_Request(policy_version=5)) + adapter = self._adapter() + await adapter.bind_weight_sync() + version = await adapter.weight_sync(_Request(policy_version=5)) self.assertEqual(version, 5) - self.assertEqual(delegate._synchronizers[0].h2d_calls, 1) + self.assertEqual(adapter._synchronizers[0].h2d_calls, 1) async def test_weight_sync_without_request_bumps_version(self): - delegate = self._delegate() - await delegate.bind_weight_sync(state={"w": 1}) - self.assertEqual(await delegate.weight_sync(), 1) - self.assertEqual(await delegate.weight_sync(), 2) + adapter = self._adapter() + await adapter.bind_weight_sync() + self.assertEqual(await adapter.weight_sync(), 1) + self.assertEqual(await adapter.weight_sync(), 2) async def test_weight_sync_with_zero_version_bumps(self): - delegate = self._delegate() - await delegate.bind_weight_sync(state={"w": 1}) - self.assertEqual(await delegate.weight_sync(_Request(policy_version=0)), 1) + adapter = self._adapter() + await adapter.bind_weight_sync() + self.assertEqual(await adapter.weight_sync(_Request(policy_version=0)), 1) async def test_weight_sync_before_bind_raises(self): - delegate = self._delegate() + adapter = self._adapter() with self.assertRaisesRegex(RuntimeError, "bind_weight_sync"): - await delegate.weight_sync() + await adapter.weight_sync() + + async def test_bind_without_sampler_raises(self): + adapter = raiden_sampler_adapter.RaidenSamplerAdapter( + server_id="test_sampler" + ) + with self.assertRaisesRegex(RuntimeError, "initialize"): + await adapter.bind_weight_sync() async def test_post_weight_sync_returns_true(self): - delegate = self._delegate() - self.assertTrue(await delegate.post_weight_sync()) + adapter = self._adapter() + self.assertTrue(await adapter.post_weight_sync()) if __name__ == "__main__": - absltest.main() + absltest.main() \ No newline at end of file diff --git a/tests/experimental/rollout/vanilla_sampler_adapter_test.py b/tests/experimental/rollout/vanilla_sampler_adapter_test.py index ba8f62252..8e97cc97a 100644 --- a/tests/experimental/rollout/vanilla_sampler_adapter_test.py +++ b/tests/experimental/rollout/vanilla_sampler_adapter_test.py @@ -15,14 +15,11 @@ """Tests for VanillaSamplerAdapter with Tunix JAX Sampler.""" import asyncio -import types -from unittest import mock from absl.testing import absltest from flax import nnx import numpy as np from tunix.experimental.rollout import sampler as base_sampler_lib from tunix.experimental.rollout import vanilla_sampler_adapter -from tunix.experimental.weight_sync import weight_sync from tunix.generate import sampler as generate_sampler_lib from tunix.tests import test_common as tc @@ -141,71 +138,9 @@ def test_uninitialized_sampler_raises(self): ) ) - def test_weight_sync_without_raiden_delegate(self): - self.assertIsNone(asyncio.run(self.vanilla_sampler.bind_weight_sync())) - self.assertTrue(asyncio.run(self.vanilla_sampler.pre_weight_sync())) - new_state = self.vanilla_sampler.sampler.transformer_state - req = base_sampler_lib.WeightSyncRequest(weights=new_state) - self.assertTrue( - asyncio.run(self.vanilla_sampler.weight_sync(sync_request=req)) - ) - self.assertTrue(asyncio.run(self.vanilla_sampler.post_weight_sync())) - with self.assertRaises(NotImplementedError): - asyncio.run(self.vanilla_sampler.get_weight_sync_metadata()) - - def test_weight_sync_with_raiden_delegate(self): - mock_delegate = mock.MagicMock() - mock_delegate.is_bounded.return_value = False - mock_delegate.bind_weight_sync = mock.AsyncMock(return_value=True) - mock_delegate.get_weight_sync_metadata = mock.AsyncMock( - return_value=[{"unit": "rollout"}] - ) - mock_delegate.pre_weight_sync = mock.AsyncMock(return_value=True) - mock_delegate.weight_sync = mock.AsyncMock(return_value=10) - mock_delegate.post_weight_sync = mock.AsyncMock(return_value=True) - - sampler_with_raiden = vanilla_sampler_adapter.VanillaSamplerAdapter( - server_id="tpu_slice_raiden", - transformer=self.transformer, - tokenizer=self.vocab, - cache_config=self.cache_config, - config=types.SimpleNamespace( - weight_sync_mode=weight_sync.WeightSyncMode.RAIDEN - ), - raiden_sync_delegate=mock_delegate, - ) - sampler_with_raiden.initialize() - - sync_req = base_sampler_lib.WeightSyncRequest(policy_version=10) - - # 1. bind_weight_sync - asyncio.run(sampler_with_raiden.bind_weight_sync(sync_req)) - mock_delegate.bind_weight_sync.assert_awaited_once_with( - sync_request=sync_req, - state=sampler_with_raiden.sampler.transformer_state, - ) - mock_delegate.is_bounded.return_value = True - - # 2. get_weight_sync_metadata - metadata = asyncio.run(sampler_with_raiden.get_weight_sync_metadata()) - self.assertEqual(metadata, [{"unit": "rollout"}]) - - # 3. pre_weight_sync - self.assertTrue(asyncio.run(sampler_with_raiden.pre_weight_sync(sync_req))) - mock_delegate.pre_weight_sync.assert_awaited_once_with( - sync_request=sync_req - ) - - # 4. weight_sync - version = asyncio.run(sampler_with_raiden.weight_sync(sync_req)) - self.assertEqual(version, 10) - mock_delegate.weight_sync.assert_awaited_once_with(sync_request=sync_req) - - # 5. post_weight_sync - self.assertTrue(asyncio.run(sampler_with_raiden.post_weight_sync(sync_req))) - mock_delegate.post_weight_sync.assert_awaited_once_with( - sync_request=sync_req - ) + def test_sample_none_requests_raises(self): + with self.assertRaises(ValueError): + asyncio.run(self.vanilla_sampler.sample(None)) if __name__ == "__main__": diff --git a/tests/experimental/worker/abstract_worker_test.py b/tests/experimental/worker/abstract_worker_test.py index 9a36689d7..3fa05c28e 100644 --- a/tests/experimental/worker/abstract_worker_test.py +++ b/tests/experimental/worker/abstract_worker_test.py @@ -69,7 +69,6 @@ class AbstractWorkerTest(parameterized.TestCase): ), kwargs=dict( worker_id="w2", - sampler=mocks.MockBaseSamplerImpl(sampler_name="mock_sampler"), tokenizer=mocks.MockTokenizer(), chat_parser=mocks.MockChatParser(), ), diff --git a/tests/experimental/worker/rollout_worker_weight_sync_test.py b/tests/experimental/worker/rollout_worker_weight_sync_test.py index 62145d4a1..ec0d2ea4b 100644 --- a/tests/experimental/worker/rollout_worker_weight_sync_test.py +++ b/tests/experimental/worker/rollout_worker_weight_sync_test.py @@ -18,7 +18,6 @@ from absl.testing import absltest from tunix.experimental.common import datatypes -from tunix.experimental.common import test_utils as mocks from tunix.experimental.worker import rollout_worker as rollout_worker_lib WorkerState = datatypes.WorkerState @@ -31,27 +30,22 @@ def __init__(self): self.admission_open = True async def pre_weight_sync(self, sync_request=None, **kwargs): - del sync_request, kwargs self.calls.append("pre") return "ok" async def weight_sync(self, sync_request=None, **kwargs): - del sync_request, kwargs self.calls.append("sync") return 1 async def post_weight_sync(self, sync_request=None, **kwargs): - del sync_request, kwargs self.calls.append("post") return "ok" - + async def bind_weight_sync(self, **kwargs): - del kwargs self.calls.append("bind") return None async def get_weight_sync_metadata(self, **kwargs): - del kwargs self.calls.append("metadata") return [{"unit": "u0"}] @@ -73,10 +67,7 @@ class WeightSyncPhasesTest(unittest.IsolatedAsyncioTestCase): def _worker(self): worker = rollout_worker_lib.RolloutWorker( - worker_id="w0", - sampler=mocks.MockBaseSamplerImpl(sampler_name="mock_sampler"), - tokenizer="mock", - chat_parser="mock", + worker_id="w0", tokenizer="mock", chat_parser="mock" ) worker.manager = _FakeManager() worker._state = WorkerState.READY @@ -130,9 +121,7 @@ async def test_full_round_call_order(self): await worker.pre_weight_sync(req) await worker.weight_sync(req) await worker.post_weight_sync(req) - self.assertEqual( - worker.manager.calls, ["bind", "metadata", "pre", "sync", "post"] - ) + self.assertEqual(worker.manager.calls, ["bind", "metadata", "pre", "sync", "post"]) async def test_bind_delegates_to_manager(self): worker = self._worker() diff --git a/tunix/experimental/common/datatypes.py b/tunix/experimental/common/datatypes.py index a739a248f..bb355156a 100644 --- a/tunix/experimental/common/datatypes.py +++ b/tunix/experimental/common/datatypes.py @@ -37,9 +37,8 @@ Role = common_datatypes.Role -# TODO(tunix-dev): Unify this extended TrajectoryItem back into -# agent_types.TrajectoryItem so that all agentic workflows share the same strict -# token array fields. +# TODO: Unify this extended TrajectoryItem back into agent_types.TrajectoryItem +# so that all agentic workflows share the same strict token array fields. @dataclasses.dataclass(kw_only=True) class TrajectoryItem(agent_types.TrajectoryItem): """Extended TrajectoryItem for Orchestrator with token arrays.""" @@ -49,6 +48,7 @@ class TrajectoryItem(agent_types.TrajectoryItem): policy_version: int = 0 + ##### Common DTOs (Data Transfer Objects) ##### @@ -397,7 +397,6 @@ class WeightSyncRequest(Request): controller_id: Optional identifier for transport controllers (e.g., TPU Raiden). policy_version: Target policy version identifier of the weights to sync. - weights: Optional source weights payload for non-Raiden / fallback sync. source_metadata: Optional transport/layout metadata describing source weights. extra_config: Optional backend-specific configuration parameters. @@ -405,7 +404,6 @@ class WeightSyncRequest(Request): controller_id: str = "" policy_version: int = 0 - weights: Any = None source_metadata: Any = None extra_config: dict[str, Any] = dataclasses.field(default_factory=dict) @@ -505,8 +503,7 @@ class SFTTrainerPayload(TrainerPayload): token_mask: ArrayLike -# TODO(tunix-dev): Introduce PPOTrainerPayload to replace generic -# RLTrainerPayload when PPO specific fields are needed. +# TODO: Introduce PPOTrainerPayload to replace generic RLTrainerPayload when PPO specific fields are needed. @dataclasses.dataclass(kw_only=True) class RLTrainerPayload(TrainerPayload): """RL training payload. @@ -532,8 +529,7 @@ class RLTrainerPayload(TrainerPayload): advantages: ArrayLike loss_mask: ArrayLike action_mask: ArrayLike | None = None - # TODO(tunix-dev): make prompt_ids/mask and completion_ids/mask required after - # SequencePackedBatchAssembler refactor is done. + # TODO(tunix-dev): make prompt_ids/mask and completion_ids/mask required after SequencePackedBatchAssembler refactor is done. prompt_ids: ArrayLike | None = None prompt_mask: ArrayLike | None = None completion_ids: ArrayLike | None = None @@ -544,7 +540,7 @@ class RLTrainerPayload(TrainerPayload): returns: ArrayLike | None = None old_values: ArrayLike | None = None metadata: dict[str, Any] = dataclasses.field(default_factory=dict) - # TODO(tunix-dev): add ppo specific fields in PPORLTrainerPayload. + # TODO: add ppo sepcific fields in a PPO specific fields in PPORLTrainerPayload @dataclasses.dataclass(kw_only=True) diff --git a/tunix/experimental/examples/math_gsm8k_dist/run_rollout_node.py b/tunix/experimental/examples/math_gsm8k_dist/run_rollout_node.py index b180ef81e..6d004e42b 100644 --- a/tunix/experimental/examples/math_gsm8k_dist/run_rollout_node.py +++ b/tunix/experimental/examples/math_gsm8k_dist/run_rollout_node.py @@ -31,9 +31,7 @@ from tunix.experimental.examples.math_gsm8k_dist import gsm8k from tunix.experimental.examples.math_gsm8k_dist import models from tunix.experimental.rollout import inprocess_vllm_sampler_adapter -from tunix.experimental.rollout import vanilla_sampler_adapter -from tunix.experimental.weight_sync import raiden_weight_sync_delegate -from tunix.experimental.weight_sync import weight_sync +from tunix.experimental.rollout import raiden_sampler_adapter from tunix.experimental.worker import remote_execution from tunix.experimental.worker import rollout_worker from tunix.generate import mappings as mappings_lib @@ -61,13 +59,9 @@ def _chat_parser_for(model_id: str, tokenizer): for family, parser_cls in CHAT_PARSERS.items(): if family in name: return parser_cls(tokenizer, enable_thinking=False) - return chat_parser_lib.DefaultChatTemplateParser( - tokenizer, enable_thinking=False - ) - + return chat_parser_lib.DefaultChatTemplateParser(tokenizer, enable_thinking=False) def _parse_args(argv: list[str]) -> argparse.Namespace: - """Parses command line arguments for the rollout worker process.""" parser = argparse.ArgumentParser(description="vLLM rollout worker process") parser.add_argument("--port", type=int, default=20001) parser.add_argument("--worker_id", type=str, default="vllm-rollout-0") @@ -87,17 +81,8 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument( "--sampler", type=str, - default=os.getenv("SAMPLER", "inprocess_vllm"), - choices=["inprocess_vllm", "vanilla"], - ) - parser.add_argument( - "--weight_sync_mode", - type=weight_sync.WeightSyncMode, - default=weight_sync.WeightSyncMode( - os.getenv("WEIGHT_SYNC_MODE", "raiden") - ), - choices=list(weight_sync.WeightSyncMode), - help="Weight sync mode (e.g. raiden, fallback).", + default=os.getenv("SAMPLER", "legacy_vllm"), + choices=["legacy_vllm", "vanilla"], ) return parser.parse_args(argv) @@ -107,44 +92,35 @@ def _create_rollout_mesh() -> Mesh: devices = mesh_utils.create_device_mesh(shape, jax.devices()) return Mesh(devices, axis_names=("fsdp", "tp")) - def _create_vanilla_worker(args, tokenizer): - """Creates a vanilla sampler rollout worker instance.""" logging.info("Creating native sampler on the rollout mesh...") mesh = _create_rollout_mesh() with mesh: model = models.create_model( args.model_name, args.model_dir or args.model_id, mesh ) - raiden_delegate = ( - raiden_weight_sync_delegate.RaidenWeightSyncDelegate() - if args.weight_sync_mode == weight_sync.WeightSyncMode.RAIDEN - else None - ) - config = rollout_worker.RolloutConfig( - sampler_type="vanilla", - weight_sync_mode=args.weight_sync_mode, - max_prompt_length=args.max_prompt_length, - max_tokens_to_generate=args.max_response_length, - temperature=1.0, - top_p=1.0, - return_logprobs=True, - env_name=gsm8k.GSM8K_ENV_NAME, - agent_name=gsm8k.GSM8K_AGENT_NAME, - ) - sampler_adapter = vanilla_sampler_adapter.VanillaSamplerAdapter( + sampler_adapter = raiden_sampler_adapter.RaidenSamplerAdapter( server_id=args.worker_id, transformer=model, tokenizer=tokenizer, cache_config=args.max_prompt_length + args.max_response_length, - config=config, - raiden_sync_delegate=raiden_delegate, ) rollout_tokenizer = tokenizer_adapter_lib.TokenizerAdapter(tokenizer) chat_parser = chat_parser_lib.QwenChatTemplateParser( tokenizer, enable_thinking=False ) + # TODO: select the chat template parser by model family instead of hardcoding. + config = rollout_worker.RolloutConfig( + sampler_type="raiden_vanilla", + max_prompt_length=args.max_prompt_length, + max_tokens_to_generate=args.max_response_length, + temperature=1.0, + top_p=1.0, + return_logprobs=True, + env_name=gsm8k.GSM8K_ENV_NAME, + agent_name=gsm8k.GSM8K_AGENT_NAME, + ) return rollout_worker.RolloutWorker( worker_id=args.worker_id, config=config, @@ -156,7 +132,6 @@ def _create_vanilla_worker(args, tokenizer): def _create_vllm_worker(args, tokenizer): - """Creates an in-process vLLM sampler rollout worker instance.""" logging.info("Creating vLLM mapping config...") mapping_config = mappings_lib.MappingConfig( lora_to_hf_mappings=mapping_vllm_jax.LORA_TO_HF_MAPPINGS @@ -172,34 +147,29 @@ def _create_vllm_worker(args, tokenizer): jax.device_count(), max_model_len, ) - lora_config = None - if args.use_lora: - lora_config = { - "max_lora_rank": args.lora_rank, - "max_loras": 1, - } vllm_config = vllm_sampler.VllmConfig( mesh=rollout_mesh, tensor_parallel_size=jax.device_count(), data_parallel_size=1, return_logprobs=True, - lora_config=lora_config, + lora_config=( + { + "max_lora_rank": args.lora_rank, + "max_loras": 1, + } + if args.use_lora + else None + ), mapping_config=mapping_config, engine_kwargs={ "model": vllm_model, "max_model_len": max_model_len, }, ) - raiden_delegate = ( - raiden_weight_sync_delegate.RaidenWeightSyncDelegate() - if args.weight_sync_mode == weight_sync.WeightSyncMode.RAIDEN - else None - ) sampler_adapter = inprocess_vllm_sampler_adapter.InprocessVllmSamplerAdapter( server_id=args.worker_id, tokenizer=tokenizer, config=vllm_config, - raiden_sync_delegate=raiden_delegate, ) rollout_tokenizer = tokenizer_adapter_lib.TokenizerAdapter(tokenizer) chat_parser = chat_parser_lib.QwenChatTemplateParser( @@ -208,7 +178,6 @@ def _create_vllm_worker(args, tokenizer): logging.info("Creating RolloutWorker wrapper...") config = rollout_worker.RolloutConfig( sampler_type="inprocess_vllm", - weight_sync_mode=args.weight_sync_mode, max_prompt_length=args.max_prompt_length, max_tokens_to_generate=args.max_response_length, temperature=1.0, @@ -254,11 +223,10 @@ def main(argv: list[str], context: Any = None) -> None: sys.path.insert(0, REPO_ROOT) logging.info("Repo root inserted into sys.path: %s", REPO_ROOT) + tokenizer_path = args.tokenizer_path or args.model_dir or args.model_id logging.info("Loading tokenizer from %s...", tokenizer_path) - tokenizer: Any = AutoTokenizer.from_pretrained( - tokenizer_path, trust_remote_code=True - ) + tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True) if tokenizer.pad_token_id is None and tokenizer.eos_token is not None: tokenizer.pad_token = tokenizer.eos_token diff --git a/tunix/experimental/rollout/inprocess_vllm_sampler_adapter.py b/tunix/experimental/rollout/inprocess_vllm_sampler_adapter.py index 2b23c3f63..e986d837c 100644 --- a/tunix/experimental/rollout/inprocess_vllm_sampler_adapter.py +++ b/tunix/experimental/rollout/inprocess_vllm_sampler_adapter.py @@ -17,11 +17,9 @@ import abc import numbers from typing import Any, List, Sequence -from absl import logging import numpy as np + from tunix.experimental.rollout import sampler as base_sampler_lib -from tunix.experimental.weight_sync import raiden_weight_sync_delegate -from tunix.experimental.weight_sync import weight_sync Sampler = base_sampler_lib.Sampler @@ -41,7 +39,6 @@ def __init__( tokenizer: Any = None, config: Any = None, model_name: str = "", - raiden_sync_delegate: Any = None, **kwargs, ): self.server_id = server_id @@ -49,32 +46,6 @@ def __init__( self.config = config self.model_name = model_name or kwargs.get("model", "") self.vllm_sampler = None - self.raiden_sync_delegate = raiden_sync_delegate - self.weight_sync_mode = getattr( - config, "weight_sync_mode", weight_sync.WeightSyncMode.FALLBACK - ) - self.enable_raiden = ( - self.weight_sync_mode == weight_sync.WeightSyncMode.RAIDEN - ) - - if self.enable_raiden: - logging.info( - "InprocessVllmSamplerAdapter [%s] weight_sync: initializing Raiden" - " delegate", - self.server_id, - ) - - if self.raiden_sync_delegate is None: - self.raiden_sync_delegate = ( - raiden_weight_sync_delegate.RaidenWeightSyncDelegate() - ) - - if not self.enable_raiden and self.raiden_sync_delegate: - logging.warning( - "InprocessVllmSamplerAdapter [%s] raiden_sync_delegate is set but" - " enable_raiden is False.", - self.server_id, - ) if self.tokenizer is not None and self.config is not None: vllm_lib = _get_vllm_sampler_cls() @@ -102,6 +73,11 @@ def initialize(self) -> None: self.vllm_sampler = vllm_lib.VllmSampler( tokenizer=self.tokenizer, config=self.config ) + if self.vllm_sampler is None: + raise RuntimeError( + f"InprocessVllmSamplerAdapter [{self.server_id}] requires a" + " vllm_sampler instance or tokenizer + config." + ) def _unpadded_prompt_tokens(self, padded_tokens: Any) -> np.ndarray: """Returns sampler-tokenized prompt ids without backend left padding.""" @@ -150,7 +126,6 @@ async def start(self, **kwargs) -> str | None | Any: return True async def stop(self, **kwargs) -> str | None | Any: - """Terminates sampler execution and closes local connections.""" del kwargs if self.vllm_sampler and hasattr(self.vllm_sampler, "stop"): self.vllm_sampler.stop() @@ -180,7 +155,6 @@ async def sample( base_sampler_lib.SamplingRequest | Sequence[base_sampler_lib.SamplingRequest] | Any - | Sequence[Any] ), **kwargs, ) -> ( @@ -298,121 +272,49 @@ async def sample( return responses[0] # --- Weight Synchronization --- - def _check_weight_sync_boundness( - self, - ): - """Returns whether the weight sync delegate is bounded.""" - if not self.enable_raiden: - return - - if not self.raiden_sync_delegate.is_bounded(): - raise RuntimeError( - f"InprocessVllmSamplerAdapter [{self.server_id}] weight sync delegate" - " is not bounded." - ) - async def get_weight_sync_metadata(self, **kwargs) -> Any: """Returns sharding specs and layout metadata across devices for weights.""" - self._check_weight_sync_boundness() - - if self.enable_raiden: - return await self.raiden_sync_delegate.get_weight_sync_metadata(**kwargs) + del kwargs raise NotImplementedError( - f"InprocessVllmSamplerAdapter [{self.server_id}] does not support" - " get_weight_sync_metadata when Raiden is disabled." + "get_weight_sync_metadata() not implemented for this SamplerServer." ) - - async def bind_weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> Any: - """Binds destination-side transport resources.""" - if self.enable_raiden: - if not hasattr(self.vllm_sampler, "transformer_state"): - raise RuntimeError( - f"InprocessVllmSamplerAdapter [{self.server_id}] does not expose" - " transformer_state for Raiden weight sync." - ) - - if self.raiden_sync_delegate.is_bounded(): - raise RuntimeError( - f"InprocessVllmSamplerAdapter [{self.server_id}] weight sync" - " delegate is already bounded before bind_weight_sync." - ) - - state = self.vllm_sampler.transformer_state - return await self.raiden_sync_delegate.bind_weight_sync( - sync_request=sync_request, state=state, **kwargs - ) + + async def bind_weight_sync(self, **kwargs) -> Any: + del kwargs return None - async def pre_weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> str | None | Any: + async def pre_weight_sync(self, sync_request: Any = None, **kwargs) -> Any: """Prepares staging handshake prior to policy weight update.""" - self._check_weight_sync_boundness() - - if self.enable_raiden: - return await self.raiden_sync_delegate.pre_weight_sync( - sync_request=sync_request, **kwargs - ) + del sync_request, kwargs return True - async def weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> str | None | Any: + async def weight_sync(self, sync_request: Any = None, **kwargs) -> Any: """Updates model weights in-place from the specified controller.""" - self._check_weight_sync_boundness() - - if self.enable_raiden: - return await self.raiden_sync_delegate.weight_sync( - sync_request=sync_request, **kwargs - ) - else: - if sync_request is None: - raise ValueError( - f"InprocessVllmSamplerAdapter Fallback mode [{self.server_id}]" - " weight_sync: sync_request is None." - ) - if self.vllm_sampler and hasattr(self.vllm_sampler, "update_params"): - weights = getattr(sync_request, "weights", None) - if weights is None: - raise ValueError( - f"InprocessVllmSamplerAdapter [{self.server_id}] weight_sync:" - " weights not found in sync_request." - ) - - self.vllm_sampler.update_params(weights) - else: - raise RuntimeError( - f"InprocessVllmSamplerAdapter [{self.server_id}] does not support" - " Raiden weight sync, while the fallback path missing required" - " components." - ) - return True - - async def post_weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> str | None | Any: - """Finalizes and switches active policy weights after transfer completion.""" - if self.enable_raiden: - return await self.raiden_sync_delegate.post_weight_sync( - sync_request=sync_request, **kwargs - ) + del kwargs + if ( + sync_request is not None + and self.vllm_sampler + and hasattr(self.vllm_sampler, "update_params") + ): + weights = getattr(sync_request, "weights", sync_request) + self.vllm_sampler.update_params(weights) return True - async def get_transfer_status(self, req_id: str | Any, **kwargs) -> str | Any: + async def get_transfer_status(self, req_id: Any, **kwargs) -> Any: """Queries status of an ongoing weight transfer or KV-cache migration.""" del req_id, kwargs return "SUCCESS" + async def get_load_info(self, **kwargs) -> base_sampler_lib.LoadInfo: + """Returns best-effort vLLM queue/cache load information.""" + del kwargs + return base_sampler_lib.LoadInfo() + + async def post_weight_sync(self, sync_request: Any = None, **kwargs) -> Any: + """Finalizes and switches active policy weights after transfer completion.""" + del sync_request, kwargs + return True + async def migrate_kv_cache( self, source_server_id: str, @@ -421,10 +323,5 @@ async def migrate_kv_cache( **kwargs, ) -> bool: """Triggers KV-cache transfer across TPU slices.""" - del source_server_id, target_server_id, token_ids, kwargs + del source_server_id, target_server_id, token_ids return True - - async def get_load_info(self, **kwargs) -> base_sampler_lib.LoadInfo: - """Returns best-effort vLLM queue/cache load information.""" - del kwargs - return base_sampler_lib.LoadInfo() diff --git a/tunix/experimental/rollout/manager.py b/tunix/experimental/rollout/manager.py index 4c14ce7bd..885e39b06 100644 --- a/tunix/experimental/rollout/manager.py +++ b/tunix/experimental/rollout/manager.py @@ -22,7 +22,6 @@ from tunix.experimental.rollout import sampler as sampler_lib from tunix.experimental.rollout import vanilla_sampler_adapter from tunix.experimental.trajectory import trajectory as trajectory_lib -from tunix.experimental.weight_sync import weight_sync from tunix.experimental.worker import traffic_controller as traffic_controller_lib from tunix.rl.rollout import base_rollout @@ -49,62 +48,32 @@ def __init__( chat_parser: Any = None, drain_timeout_s: float = 300.0, ): - """Initializes the RolloutManager. - - Args: - config: RolloutConfig configuration options. - sampler: Optional pre-constructed Sampler instance. - env_pool: Environment pool for rollout execution. - agent_factory: Factory callable producing agent instances. - max_concurrency: Maximum number of concurrent episodes. - tokenizer: Tokenizer for prompt/response encoding. - chat_parser: Chat parser for conversation templating. - drain_timeout_s: How long pre_weight_sync waits for in-flight trajectories - before pausing the stragglers, roughly one worst-case trajectory. - """ + """drain_timeout_s: how long pre_weight_sync waits for in-flight + trajectories before pausing the stragglers, roughly one worst-case + trajectory.""" self.config = config if sampler is None: sampler_type = getattr(config, "sampler_type", "vanilla") - weight_sync_mode = getattr( - config, "weight_sync_mode", weight_sync.WeightSyncMode.FALLBACK - ) - if sampler_type == "vllm": raise NotImplementedError( "vLLM sampler is not implemented yet. Use 'inprocess_vllm' or" " 'vanilla'." ) - elif "inprocess_vllm" in sampler_type: + elif sampler_type == "inprocess_vllm": from tunix.experimental.rollout import inprocess_vllm_sampler_adapter # pylint: disable=g-import-not-at-top - raiden_delegate = None - if weight_sync_mode == weight_sync.WeightSyncMode.RAIDEN: - from tunix.experimental.weight_sync import raiden_weight_sync_delegate # pylint: disable=g-import-not-at-top - - raiden_delegate = ( - raiden_weight_sync_delegate.RaidenWeightSyncDelegate() - ) - sampler = inprocess_vllm_sampler_adapter.InprocessVllmSamplerAdapter( # pyrefly: ignore[bad-instantiation] server_id="inprocess_vllm_sampler", - tokenizer=tokenizer, - config=config, - raiden_sync_delegate=raiden_delegate, ) - elif "vanilla" in sampler_type: - raiden_delegate = None - if weight_sync_mode == weight_sync.WeightSyncMode.RAIDEN: - from tunix.experimental.weight_sync import raiden_weight_sync_delegate # pylint: disable=g-import-not-at-top - - raiden_delegate = ( - raiden_weight_sync_delegate.RaidenWeightSyncDelegate() - ) - - sampler = vanilla_sampler_adapter.VanillaSamplerAdapter( + elif sampler_type == "vanilla": + sampler = vanilla_sampler_adapter.VanillaSamplerAdapter( # pyrefly: ignore[bad-instantiation] server_id="vanilla_sampler", - tokenizer=tokenizer, - config=config, - raiden_sync_delegate=raiden_delegate, + ) + elif sampler_type == "raiden_vanilla": + from tunix.experimental.rollout import raiden_sampler_adapter # pylint: disable=g-import-not-at-top + + sampler = raiden_sampler_adapter.RaidenSamplerAdapter( # pyrefly: ignore[bad-instantiation] + server_id="raiden_vanilla_sampler", ) else: raise ValueError(f"Unknown sampler_type: {sampler_type}") @@ -332,7 +301,7 @@ async def post_weight_sync( self.resume_all() self._traffic.reopen() return res - + def reopen_admission(self) -> bool: """Reopens rollout admission after an aborted round.""" return self._traffic.reopen() diff --git a/tunix/experimental/weight_sync/raiden_weight_sync_delegate.py b/tunix/experimental/rollout/raiden_sampler_adapter.py similarity index 75% rename from tunix/experimental/weight_sync/raiden_weight_sync_delegate.py rename to tunix/experimental/rollout/raiden_sampler_adapter.py index 61546f2d1..7cec84e06 100644 --- a/tunix/experimental/weight_sync/raiden_weight_sync_delegate.py +++ b/tunix/experimental/rollout/raiden_sampler_adapter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Raiden weight sync delegate for destination-side rollout workers.""" +"""Vanilla sampler adapter with raiden weight sync as the destination.""" from __future__ import annotations @@ -20,11 +20,13 @@ from typing import Any, List from absl import logging + +from tunix.experimental.rollout import vanilla_sampler_adapter from tunix.experimental.weight_sync import raiden_synchronizer -class RaidenWeightSyncDelegate: - """Manages weight synchronization over Raiden for sampler adapters. +class RaidenSamplerAdapter(vanilla_sampler_adapter.VanillaSamplerAdapter): + """Serves with the in-process sampler; syncs weights over raiden. The destination side of a weight sync round: bind_weight_sync binds the sampler's transformer state to the raiden transport, the transfer lands @@ -45,37 +47,30 @@ def __init__(self, *args, worker_index: int = 0, **kwargs): ] self._version = 0 - def is_bounded( - self, - ) -> bool: - """Returns whether all managed synchronizers are bound.""" - return all(s.bound for s in self._synchronizers) - - async def bind_weight_sync( - self, sync_request: Any = None, state: Any = None, **kwargs - ) -> Any: - """Binds destination-side transport resources for weight sync.""" - del sync_request, kwargs - + def _bound_synchronizers(self) -> List[Any]: + if self.sampler is None: + raise RuntimeError("initialize the sampler before weight sync") for sync in self._synchronizers: # The state arrays never change, so one bind covers every round. if not sync.bound: - sync.bind(state) + sync.bind(self.sampler.transformer_state) + return self._synchronizers - return True + async def bind_weight_sync(self, sync_request: Any = None, **kwargs) -> Any: + del sync_request, kwargs + self._bound_synchronizers() + return None async def get_weight_sync_metadata(self, **kwargs) -> Any: - """Retrieves destination worker metadata for the sync coordinator.""" del kwargs - return [s.work_unit_metadata() for s in self._synchronizers] + return [s.work_unit_metadata() for s in self._bound_synchronizers()] async def pre_weight_sync(self, sync_request: Any = None, **kwargs) -> Any: - """Pre-sync phase hook executed before weight transfer begins.""" del sync_request, kwargs + self._bound_synchronizers() return True async def weight_sync(self, sync_request: Any = None, **kwargs) -> Any: - """Executes weight installation on device from host staging buffer.""" del kwargs for sync in self._synchronizers: if not sync.bound: @@ -90,9 +85,8 @@ async def weight_sync(self, sync_request: Any = None, **kwargs) -> Any: return self._version async def post_weight_sync(self, sync_request: Any = None, **kwargs) -> Any: - """Post-sync phase hook executed after weight installation completes.""" del sync_request, kwargs if os.environ.get("VERIFY_WEIGHTS", "").lower() == "true": for sync in self._synchronizers: logging.info("raiden metrics: %s", sync.metrics()) - return True + return True \ No newline at end of file diff --git a/tunix/experimental/rollout/sampler.py b/tunix/experimental/rollout/sampler.py index 88ec51f1c..7a6b86661 100644 --- a/tunix/experimental/rollout/sampler.py +++ b/tunix/experimental/rollout/sampler.py @@ -19,9 +19,8 @@ from jax import typing import numpy as np -from tunix.experimental.common import datatypes - ArrayLike = typing.ArrayLike +from tunix.experimental.common import datatypes @dataclasses.dataclass(kw_only=True) @@ -114,7 +113,6 @@ class WeightSyncRequest(datatypes.Request): controller_id: Optional identifier for transport controllers (e.g., TPU Raiden). policy_version: Target policy version identifier of the weights to sync. - weights: Optional source weights payload for non-Raiden / fallback sync. source_metadata: Optional transport/layout metadata describing source weights. extra_config: Optional backend-specific configuration parameters. @@ -122,7 +120,6 @@ class WeightSyncRequest(datatypes.Request): controller_id: str = "" policy_version: int = 0 - weights: Any = None source_metadata: Any = None extra_config: dict[str, Any] = dataclasses.field(default_factory=dict) @@ -177,20 +174,16 @@ async def sample( ) -> list[SamplingResponse] | Any: """Generates completions for a batch of prompt conversations concurrently.""" ... + + async def bind_weight_sync(self, **kwargs) -> Any: + """Binds destination-side transport resources. Idempotent per round.""" + ... # --- Weight Synchronization --- async def get_weight_sync_metadata(self, **kwargs) -> Any: """Returns the sharding specs and layout metadata across devices for policy model weights.""" ... - async def bind_weight_sync( - self, - sync_request: WeightSyncRequest | Any = None, - **kwargs, - ) -> Any: - """Binds destination-side transport resources. Idempotent per round.""" - ... - async def pre_weight_sync( self, sync_request: WeightSyncRequest | Any = None, **kwargs ) -> str | None | Any: diff --git a/tunix/experimental/rollout/vanilla_sampler_adapter.py b/tunix/experimental/rollout/vanilla_sampler_adapter.py index 39c127bf9..338f075cc 100644 --- a/tunix/experimental/rollout/vanilla_sampler_adapter.py +++ b/tunix/experimental/rollout/vanilla_sampler_adapter.py @@ -17,11 +17,8 @@ import abc import numbers from typing import Any, List, Sequence -from absl import logging import numpy as np from tunix.experimental.rollout import sampler as base_sampler_lib -from tunix.experimental.weight_sync import raiden_weight_sync_delegate -from tunix.experimental.weight_sync import weight_sync from tunix.generate import sampler as generate_sampler_lib Sampler = base_sampler_lib.Sampler @@ -32,17 +29,6 @@ class VanillaSamplerAdapter(Sampler, abc.ABC): Constructs or wraps a Tunix generate_sampler_lib.Sampler instance and executes sampling requests. - - Supported Weight Synchronization Modes: - 1. Raiden Mode (`weight_sync_mode == RAIDEN`): - Delegates high-performance decentralized P2P / DCN weight synchronization - to `RaidenWeightSyncDelegate`. Binds destination memory buffers using - `self.sampler.transformer_state` and executes phased synchronization - lifecycle hooks (`bind`, `pre`, `sync`, `post`). - 2. Fallback Direct Mode (`weight_sync_mode == FALLBACK`): - Synchronizes weights in-place without Raiden transport. When a weight - payload is received in `sync_request.weights`, updates the underlying - JAX sampler directly via `self.sampler.transformer_state`. """ def __init__( @@ -53,34 +39,12 @@ def __init__( cache_config: generate_sampler_lib.CacheConfig | int | None = None, image_processor: Any = None, model: Any = None, - config: Any = None, - raiden_sync_delegate: Any = None, **kwargs, ): self.server_id = server_id self.transformer = transformer if transformer is not None else model self.tokenizer = tokenizer self.image_processor = image_processor - self.config = config - self.raiden_sync_delegate = raiden_sync_delegate - self.weight_sync_mode = getattr( - config, "weight_sync_mode", weight_sync.WeightSyncMode.FALLBACK - ) - self.enable_raiden = ( - self.weight_sync_mode == weight_sync.WeightSyncMode.RAIDEN - ) - - if self.enable_raiden and self.raiden_sync_delegate is None: - self.raiden_sync_delegate = ( - raiden_weight_sync_delegate.RaidenWeightSyncDelegate() - ) - - if not self.enable_raiden and self.raiden_sync_delegate: - logging.warning( - "VanillaSamplerAdapter [%s] raiden_sync_delegate is set but" - " enable_raiden is False.", - self.server_id, - ) if self.transformer is not None and self.tokenizer is not None: self.sampler = self._build_generate_sampler(cache_config) @@ -95,25 +59,30 @@ def _build_generate_sampler( cache_cfg = cache_config else: cfg = getattr(self.transformer, "config", None) - if cfg: - num_layers = getattr( - cfg, "num_layers", getattr(cfg, "num_hidden_layers", 4) - ) - num_kv_heads = getattr( - cfg, "num_kv_heads", getattr(cfg, "num_key_value_heads", 4) - ) - head_dim = getattr(cfg, "head_dim", getattr(cfg, "head_dimension", 16)) - cache_size = ( - cache_config - if isinstance(cache_config, int) - else getattr(cfg, "max_position_embeddings", 1024) - ) - else: - num_layers = 4 - num_kv_heads = 4 - head_dim = 16 - cache_size = cache_config if isinstance(cache_config, int) else 1024 - + num_layers = ( + getattr(cfg, "num_layers", getattr(cfg, "num_hidden_layers", 4)) + if cfg + else 4 + ) + num_kv_heads = ( + getattr( + cfg, "num_kv_heads", getattr(cfg, "num_key_value_heads", 4) + ) + if cfg + else 4 + ) + head_dim = ( + getattr(cfg, "head_dim", getattr(cfg, "head_dimension", 16)) + if cfg + else 16 + ) + cache_size = ( + cache_config + if isinstance(cache_config, int) + else ( + getattr(cfg, "max_position_embeddings", 1024) if cfg else 1024 + ) + ) cache_cfg = generate_sampler_lib.CacheConfig( cache_size=cache_size, num_layers=num_layers, @@ -137,6 +106,14 @@ def initialize(self) -> None: ): self.sampler = self._build_generate_sampler(None) + if self.sampler is None and ( + self.transformer is not None or self.tokenizer is not None + ): + raise RuntimeError( + f"VanillaSamplerAdapter [{self.server_id}] requires a sampler" + " instance or transformer + tokenizer." + ) + def _unpadded_prompt_tokens(self, padded_tokens: Any) -> np.ndarray: """Returns sampler-tokenized prompt ids without backend left padding.""" arr = np.asarray(padded_tokens, dtype=np.int32).reshape(-1) @@ -184,7 +161,6 @@ async def sample( base_sampler_lib.SamplingRequest | Sequence[base_sampler_lib.SamplingRequest] | Any - | Sequence[Any] ), **kwargs, ) -> ( @@ -283,9 +259,14 @@ async def sample( if isinstance(sampler_output.tokens, list) else sampler_output.tokens ) - lps = None - if sampler_output.logprobs and isinstance(sampler_output.logprobs, list): - lps = sampler_output.logprobs[i] + lps = ( + sampler_output.logprobs[i] + if ( + sampler_output.logprobs + and isinstance(sampler_output.logprobs, list) + ) + else None + ) tok_ids = ( np.array(toks, dtype=np.int32) @@ -313,144 +294,42 @@ async def sample( return responses[0] # --- Weight Synchronization --- - async def get_transfer_status(self, req_id: str | Any, **kwargs) -> str | Any: - """Queries status of an ongoing weight transfer or KV-cache migration.""" - del req_id, kwargs - return "SUCCESS" - - async def get_load_info(self, **kwargs) -> base_sampler_lib.LoadInfo: - """Returns best-effort local sampler load information.""" - del kwargs - return base_sampler_lib.LoadInfo() - - # --- Weight Synchronization --- - def _check_weight_sync_boundness( - self, - ): - """Verifies that the Raiden delegate has been bound before executing sync phases.""" - if not self.enable_raiden: - return - - if not self.raiden_sync_delegate.is_bounded(): - raise RuntimeError( - f"VanillaSamplerAdapter [{self.server_id}] weight sync delegate" - " is not bounded." - ) - async def get_weight_sync_metadata(self, **kwargs) -> Any: """Returns sharding specs and layout metadata across devices for weights.""" - self._check_weight_sync_boundness() - - # In Raiden mode, retrieve transport endpoint and tensor shard layout - # metadata. - if self.enable_raiden: - return await self.raiden_sync_delegate.get_weight_sync_metadata(**kwargs) - - # In Fallback mode, metadata query is not supported. + del kwargs raise NotImplementedError( - f"VanillaSamplerAdapter [{self.server_id}] does not support" - " get_weight_sync_metadata when Raiden is disabled." + "get_weight_sync_metadata() not implemented for this SamplerServer." ) - async def bind_weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> Any: - """Binds destination-side transport resources for weight transfer.""" - if self.enable_raiden: - # In Raiden mode, register destination sampler transformer_state memory - # buffers. - if not hasattr(self.sampler, "transformer_state"): - raise RuntimeError( - f"VanillaSamplerAdapter [{self.server_id}] sampler does not expose" - " transformer_state for Raiden weight sync." - ) - - if self.raiden_sync_delegate.is_bounded(): - raise RuntimeError( - f"VanillaSamplerAdapter [{self.server_id}] weight sync delegate is" - " already bounded before bind_weight_sync." - ) - - state = self.sampler.transformer_state - return await self.raiden_sync_delegate.bind_weight_sync( - sync_request=sync_request, state=state, **kwargs - ) - # In Fallback mode, no transport binding is required. + async def bind_weight_sync(self, **kwargs) -> Any: + del kwargs return None - async def pre_weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> str | None | Any: + async def pre_weight_sync(self, sync_request: Any = None, **kwargs) -> Any: """Prepares staging handshake prior to policy weight update.""" - self._check_weight_sync_boundness() + del sync_request, kwargs + return True - # In Raiden mode, execute the pre-synchronization barrier via delegate. - if self.enable_raiden: - return await self.raiden_sync_delegate.pre_weight_sync( - sync_request=sync_request, **kwargs - ) - # In Fallback mode, acts as a no-op returning True. + async def weight_sync(self, sync_request: Any = None, **kwargs) -> Any: + """Updates model weights in-place from the specified controller.""" + del sync_request, kwargs return True - async def weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> str | None | Any: - """Updates model weights in-place from the specified controller or request.""" - self._check_weight_sync_boundness() - - # Raiden mode: Invoke Raiden transport to stream weights into bound memory - # buffers. - if self.enable_raiden: - return await self.raiden_sync_delegate.weight_sync( - sync_request=sync_request, **kwargs - ) - else: - # Fallback mode: Directly assign source weights from sync_request.weights. - if sync_request is None: - raise ValueError( - "VanillaSamplerAdapter Fallback mode [%s] weight_sync:" - " sync_request is None." - % self.server_id - ) - if self.sampler and hasattr(self.sampler, "transformer_state"): - weights = getattr(sync_request, "weights", None) - if weights is None: - raise ValueError( - "VanillaSamplerAdapter [%s] weight_sync: weights not found" - " in sync_request." - % self.server_id - ) - self.sampler.transformer_state = weights - else: - raise RuntimeError( - f"VanillaSamplerAdapter [{self.server_id}] does not support" - " Raiden weight sync, while the fallback path missing required" - " components." - ) - return True - - async def post_weight_sync( - self, - sync_request: base_sampler_lib.WeightSyncRequest | Any = None, - **kwargs, - ) -> str | None | Any: + async def get_transfer_status(self, req_id: Any, **kwargs) -> Any: + """Queries status of an ongoing weight transfer or KV-cache migration.""" + del req_id, kwargs + return "SUCCESS" + + async def get_load_info(self, **kwargs) -> base_sampler_lib.LoadInfo: + """Returns best-effort local sampler load information.""" + del kwargs + return base_sampler_lib.LoadInfo() + + async def post_weight_sync(self, sync_request: Any = None, **kwargs) -> Any: """Finalizes and switches active policy weights after transfer completion.""" - # Raiden mode: Commit newly transferred weights and execute post-sync - # barrier. - if self.enable_raiden: - return await self.raiden_sync_delegate.post_weight_sync( - sync_request=sync_request, **kwargs - ) - # Fallback mode: acts as a no-op returning True. + del sync_request, kwargs return True - # --- KV-cache Migration --- async def migrate_kv_cache( self, source_server_id: str, @@ -459,5 +338,5 @@ async def migrate_kv_cache( **kwargs, ) -> bool: """Triggers Raiden P2P KV-cache transfer across TPU slices.""" - del source_server_id, target_server_id, token_ids, kwargs + del source_server_id, target_server_id, token_ids return True diff --git a/tunix/experimental/weight_sync/weight_sync.py b/tunix/experimental/weight_sync/weight_sync.py index 6a13adcae..e5746535a 100644 --- a/tunix/experimental/weight_sync/weight_sync.py +++ b/tunix/experimental/weight_sync/weight_sync.py @@ -27,17 +27,9 @@ import abc import dataclasses -import enum from typing import Any, Mapping, Optional, Protocol, Sequence, runtime_checkable -class WeightSyncMode(str, enum.Enum): - """Modes for weight synchronization across workers.""" - - FALLBACK = "fallback" - RAIDEN = "raiden" - - @dataclasses.dataclass(frozen=True) class WorkUnitId: """Transport-neutral identity for one participant's data work unit. @@ -69,18 +61,12 @@ class TensorMetadata: not need it may ignore it. Attributes: - name: Variable name. - shape: Shape of the tensor. - mesh_shape: Mesh shape for this tensor. - layout: Layout mapping. - item_size: Bytes per element. - layer_idx: Stable batching ordinal. sharding_spec: One mesh axis name per TENSOR dimension, empty string where - that dimension is replicated. This is the subset of JAX `PartitionSpec` - used by the Tunix/JAX adapters: `P(None, "y")` is `("", "y")`. Together - with the work unit's physical `mesh_axes`, it maps device coordinates onto - the variable's logical mesh. A concrete transport must reject forms its - wire representation cannot encode. + that dimension is replicated. This is the subset of JAX + `PartitionSpec` used by the Tunix/JAX adapters: `P(None, "y")` is + `("", "y")`. Together with the work unit's physical `mesh_axes`, it + maps device coordinates onto the variable's logical mesh. A concrete + transport must reject forms its wire representation cannot encode. """ name: str @@ -271,27 +257,16 @@ async def prepare_weight_sync( transport commonly uses one unit per physical host/listener, so a multi-host source returns several. Wire-safe values only; no device arrays. - - Args: - sync_request: Optional request context for the sync round. - **kwargs: Additional transport-specific options. """ ... async def release_weight_sync( self, sync_request: Any = None, **kwargs: Any ) -> Any: - """Releases this round's staging. - - Called on every exit path except an UNKNOWN_TRANSFER_STATE round (a - possibly-live transfer may still be reading the staging). Idempotent, and - must be safe to call while a timed-out prepare for the same round is still - running remotely. - - Args: - sync_request: Optional request context for the sync round. - **kwargs: Additional transport-specific options. - """ + """Releases this round's staging. Called on every exit path except an + UNKNOWN_TRANSFER_STATE round (a possibly-live transfer may still be + reading the staging). Idempotent, and must be safe to call while a + timed-out prepare for the same round is still running remotely.""" ... @@ -338,19 +313,13 @@ async def get_weight_sync_metadata( """ ... - async def pre_weight_sync( - self, sync_request: Any = None, **kwargs: Any - ) -> Any: + async def pre_weight_sync(self, sync_request: Any = None, **kwargs: Any) -> Any: """Quiesces the worker so the arriving weights have somewhere to land. Must actually gate admission: stop accepting new requests, drain or cancel in-flight ones, drop the prefix cache, free the KV cache. The worker is not serving from the moment this returns until post or abort. Merely setting a pause flag does not satisfy this. - - Args: - sync_request: Optional request context for the sync round. - **kwargs: Additional transport-specific options. """ ... @@ -361,31 +330,19 @@ async def weight_sync(self, sync_request: Any = None, **kwargs: Any) -> Any: performs H2D from host staging here; a file-backed worker may load its prepared checkpoint. It must not touch the serving copy and records the pending policy version for post to publish. - - Args: - sync_request: Optional request context for the sync round. - **kwargs: Additional transport-specific options. """ ... - async def post_weight_sync( - self, sync_request: Any = None, **kwargs: Any - ) -> Any: + async def post_weight_sync(self, sync_request: Any = None, **kwargs: Any) -> Any: """Publishes the pending weights atomically, rebuilds caches, resumes. Must be idempotent for the round key: a retry after a lost reply, or after a crash between publishing and recording, must converge to the same committed state rather than fail or double-apply. - - Args: - sync_request: Optional request context for the sync round. - **kwargs: Additional transport-specific options. """ ... - async def abort_weight_sync( - self, sync_request: Any = None, **kwargs: Any - ) -> Any: + async def abort_weight_sync(self, sync_request: Any = None, **kwargs: Any) -> Any: """Rolls back to serving the previous weights. Invalidates this round's staging -- physically or logically: a @@ -405,10 +362,6 @@ async def abort_weight_sync( between any two syncs, and the alternative on every such failure would be a fleet restart that lands on the same old version far more expensively. - - Args: - sync_request: Optional request context for the sync round. - **kwargs: Additional transport-specific options. """ ... @@ -418,4 +371,4 @@ async def get_weight_sync_status(self) -> Mapping[str, Any]: Consulted by the coordinator whenever a phase RPC fails, to distinguish a lost reply from unfinished work. """ - ... + ... \ No newline at end of file diff --git a/tunix/experimental/worker/rollout_worker.py b/tunix/experimental/worker/rollout_worker.py index f57e57a21..41db09c1e 100644 --- a/tunix/experimental/worker/rollout_worker.py +++ b/tunix/experimental/worker/rollout_worker.py @@ -21,7 +21,6 @@ from tunix.experimental.rollout import manager as manager_lib from tunix.experimental.rollout import sampler as sampler_lib from tunix.experimental.trajectory import trajectory as trajectory_lib -from tunix.experimental.weight_sync import weight_sync from tunix.experimental.worker import abstract_worker from tunix.rl.rollout import base_rollout @@ -33,8 +32,6 @@ class RolloutConfig(base_rollout.RolloutConfig): Attributes: sampler_type: Type of sampler adapter to construct ("vanilla", "inprocess_vllm", "vllm"). - weight_sync_mode: Mode of weight synchronization ("default", "fallback", - "raiden"). env_name: Registered name of environment class in ENV_REGISTRY. agent_name: Registered name of agent class in AGENT_REGISTRY. env_config: Configuration dictionary passed to environment constructor. @@ -42,9 +39,6 @@ class RolloutConfig(base_rollout.RolloutConfig): """ sampler_type: str = "vanilla" - weight_sync_mode: weight_sync.WeightSyncMode = ( - weight_sync.WeightSyncMode.FALLBACK - ) env_name: str = "" agent_name: str = "" env_config: dict[str, Any] = dataclasses.field(default_factory=dict) @@ -230,9 +224,7 @@ async def sample_prompts( if max_generation_steps is not None else config.max_tokens_to_generate ), - temperature=( - temperature if temperature is not None else config.temperature - ), + temperature=temperature if temperature is not None else config.temperature, top_p=top_p if top_p is not None else config.top_p, top_k=top_k if top_k is not None else config.top_k, seed=seed if seed is not None else config.seed, # pyrefly: ignore[bad-argument-type] @@ -389,9 +381,7 @@ async def _generate_rollout_requests_direct( max_tokens=sample_kwargs.get( "max_generation_steps", config.max_tokens_to_generate ), - temperature=( - sample_kwargs.get("temperature", config.temperature) - ), + temperature=sample_kwargs.get("temperature", config.temperature), top_p=sample_kwargs.get("top_p", config.top_p), top_k=sample_kwargs.get("top_k", config.top_k), seed=sample_kwargs.get("seed", config.seed),