Support v1 raw multimodal image offload

packages/prime-rl-configs/src/prime_rl/configs/env_server.py

@@ -3,7 +3,7 @@
 import verifiers.v1 as vf
 from pydantic import SerializeAsAny, model_validator
 
-from prime_rl.configs.shared import LogConfig
+from prime_rl.configs.shared import LogConfig, MultimodalConfig
 from prime_rl.utils.config import BaseConfig
 
 
@@ -26,6 +26,9 @@ class EnvServerConfig(BaseConfig):
     output_dir: Path = Path("outputs")
     """Directory to write outputs to — logs and any generated artifacts are written as subdirectories."""
 
+    multimodal: MultimodalConfig = MultimodalConfig()
+    """Raw multimodal image offload settings; with ``output_dir``, resolves the run image-asset dir this server offloads into."""
+
     @model_validator(mode="before")
     @classmethod
     def _resolve_env(cls, data):

packages/prime-rl-configs/src/prime_rl/configs/inference.py

@@ -5,7 +5,7 @@
 from pydantic import Field, model_validator
 from pydantic_config import BaseConfig
 
-from prime_rl.configs.shared import BaseModelConfig, EnvVars, LogConfig, SlurmConfig
+from prime_rl.configs.shared import BaseModelConfig, EnvVars, LogConfig, MultimodalConfig, SlurmConfig
 from prime_rl.utils.config import find_package_resource, rgetattr, rsetattr
 from prime_rl.utils.parsers import resolve_reasoning_parser, resolve_tool_call_parser
 
@@ -423,6 +423,9 @@ class InferenceConfig(BaseConfig):
     dry_run: bool = False
     """Only validate and dump resolved configs, then exit early."""
 
+    multimodal: MultimodalConfig = MultimodalConfig()
+    """Raw multimodal image offload settings shared with trainer and orchestrator."""
+
     @model_validator(mode="after")
     def validate_multi_node_requires_slurm(self):
         if self.deployment.type in ("multi_node", "disaggregated") and self.slurm is None:

packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py

@@ -17,6 +17,7 @@
     FileMonitorConfig,
     HeartbeatConfig,
     LogConfig,
+    MultimodalConfig,
     PrimeMonitorConfig,
     TransportConfig,
     WandbWithExtrasConfig,
@@ -571,6 +572,9 @@ class OrchestratorConfig(BaseConfig):
     heartbeat: HeartbeatConfig | None = None
     """BetterStack heartbeat configuration for monitoring training progress."""
 
+    multimodal: MultimodalConfig = MultimodalConfig()
+    """Raw multimodal image offload settings shared with trainer and inference."""
+
     @model_validator(mode="after")
     def auto_setup_tokenizer(self):
         if self.tokenizer.name is None:

packages/prime-rl-configs/src/prime_rl/configs/rl.py

@@ -21,6 +21,7 @@
 from prime_rl.configs.shared import (
     EnvVars,
     FileMonitorConfig,
+    MultimodalConfig,
     SlurmConfig,
     TransportConfig,
     VLMConfig,
@@ -261,6 +262,8 @@ class RLConfig(BaseConfig):
 
     weight_broadcast: SharedWeightBroadcastConfig | None = None
 
+    multimodal: MultimodalConfig = MultimodalConfig()
+    """Shared raw multimodal image offload settings. Propagated to trainer, orchestrator, and inference."""
     rollout_transport: TransportConfig | None = None
 
     bench: bool = False

packages/prime-rl-configs/src/prime_rl/configs/sft.py

@@ -248,6 +248,23 @@ def normalize_deployment(cls, data):
 
     ### Validate configs (e.g. raise for unsupported (combinations of) configs)
 
+    @model_validator(mode="after")
+    def renderer_emits_processed_multimodal(self):
+        """SFT consumes processed pixel tensors straight from the renderer — it has no
+        raw-ref materializer, so the renderers-library default of ``multimodal_output='raw'``
+        (built for the RL offload path) would silently ship JSON descriptors into training.
+        Default SFT renderers to ``'processed'`` and reject an explicit ``'raw'``."""
+        if "multimodal_output" in self.renderer.model_fields_set:
+            if self.renderer.multimodal_output == "raw":
+                raise ValueError(
+                    "multimodal_output='raw' is unsupported for SFT: the SFT data path materializes "
+                    "images from processed renderer output, not raw refs. Remove the override "
+                    "(SFT defaults to 'processed')."
+                )
+        else:
+            self.renderer = self.renderer.model_copy(update={"multimodal_output": "processed"})
+        return self
+
     @model_validator(mode="after")
     def deepep_disables_grad_clipping(self):
         if self.model.ep_comm_backend == "deepep" and self.optim.max_norm is not None:

packages/prime-rl-configs/src/prime_rl/configs/shared.py

@@ -6,19 +6,29 @@
 
 from prime_rl.utils.config import BaseConfig
 
-# Launcher-managed env vars that a component's `env_vars` must not set: GPU partitioning
-# and the single shared W&B run. The launcher always sets these last, so allowing them in
-# `env_vars` would be a silent no-op (or, on multi-node, a footgun) — reject them instead.
+# Env vars that a component's `env_vars` must not set:
+# - CUDA_VISIBLE_DEVICES / WANDB_SHARED_*: the launcher always sets these last.
+# - VF_RENDERER_IMAGE_OFFLOAD_DIR: each env-server resolves this from
+#   ``[multimodal].offload_dir`` (operator process env wins via setdefault).
+# Allowing them in ``env_vars`` would be a silent no-op or a footgun — reject instead.
 PROTECTED_ENV_VARS = frozenset(
-    {"CUDA_VISIBLE_DEVICES", "WANDB_SHARED_MODE", "WANDB_SHARED_RUN_ID", "WANDB_SHARED_LABEL"}
+    {
+        "CUDA_VISIBLE_DEVICES",
+        "VF_RENDERER_IMAGE_OFFLOAD_DIR",
+        "WANDB_SHARED_MODE",
+        "WANDB_SHARED_RUN_ID",
+        "WANDB_SHARED_LABEL",
+    }
 )
 
 
 def reject_protected_env_vars(env_vars: dict[str, str]) -> dict[str, str]:
     clobbered = sorted(PROTECTED_ENV_VARS & env_vars.keys())
     if clobbered:
         raise ValueError(
-            f"env_vars cannot set launcher-managed vars {clobbered} — set by the launcher, not overridable"
+            f"env_vars cannot set protected vars {clobbered}; "
+            "use [multimodal].offload_dir for VF_RENDERER_IMAGE_OFFLOAD_DIR, "
+            "and leave CUDA_VISIBLE_DEVICES / WANDB_SHARED_* to the launcher"
         )
     return env_vars
 
@@ -86,6 +96,11 @@ def resolve_project_dir(self):
 ServerType = Literal["vllm", "openai"]
 
 
+class MultimodalConfig(BaseConfig):
+    offload_dir: Path | None = None
+    """Directory for offloaded image assets. Supports environment expansion such as ``/data/outputs/run_${RUN_ID}/assets/images``. When unset, prime-rl resolves a run-scoped default."""
+
+
 class VLMConfig(BaseConfig):
     vision_encoder_attr: str
     """Dotted attribute path to the vision encoder module (e.g. ``model.visual``)."""

packages/prime-rl-configs/src/prime_rl/configs/trainer.py

@@ -10,6 +10,7 @@
     FileMonitorConfig,
     HeartbeatConfig,
     MetricsServerConfig,
+    MultimodalConfig,
     TrainerLogConfig,
     TransportConfig,
     WandbConfig,
@@ -628,6 +629,9 @@ class TrainerConfig(BaseConfig):
     max_concurrent_runs: int = Field(1, ge=1)
     """Maximum number of concurrent runs to allow. If 1, only one run may run at a time."""
 
+    multimodal: MultimodalConfig = MultimodalConfig()
+    """Raw multimodal image offload settings shared with orchestrator and inference."""
+
     enable_token_export: bool = False
     """Opt-in per-token JSONL export for rollout debugging. When enabled, writes token ids and aligned trainer metrics after each forward pass."""
 

packages/prime-rl-configs/src/prime_rl/utils/validation.py

@@ -132,6 +132,7 @@ def propagate(shared_path: str, *targets: str) -> None:
     # Top-level scalars.
     propagate("max_steps", "trainer.max_steps", "orchestrator.max_steps")
     propagate("seq_len", "trainer.model.seq_len", "orchestrator.seq_len")
+    propagate("multimodal", "trainer.multimodal", "orchestrator.multimodal", "inference.multimodal")
 
     # [slurm] → inference: a multi-node RL run drives its inference deployment under
     # the same SLURM allocation, so the nested inference inherits [slurm]. This is

pyproject.toml

@@ -133,9 +133,12 @@ members = [
 ]
 # tau2-synth-v1 and tau3-bench-v1 pin `tau2` forks/revs that conflict with the
 # canonical tau2-bench-v1; a workspace shares one lock, so these can't coexist.
+# nemo-gym-weather-v1 needs verifiers[nemo-gym], whose nemo-gym==0.4.0 pins
+# openai<=2.7.2 against verifiers' own openai>=2.9.0.
 exclude = [
     "deps/research-environments/environments/tool_use/tau2_synth_v1",
     "deps/research-environments/environments/tool_use/tau3_bench_v1",
+    "deps/verifiers/environments/nemo_gym_weather_v1",
 ]
 
 # ModelExpress 0.3.0 publishes protobuf<6 metadata, but its generated proto is

skills/training/monitor-run/SKILL.md

@@ -173,6 +173,37 @@ A few warnings are normal. Escalate when errors are persistent, growing, or hit
 - **Trainer**: NCCL/CUDA errors, OOM, NaN loss or gradients.
 - **Inference**: NCCL/CUDA errors, OOM, request timeouts.
 
+### Multimodal image offload checks
+
+v1 multimodal RL offloads images exactly once, at verifiers ingress: every image
+content part is rewritten to a `file://` asset under the run image directory
+before rendering, and renderers/inference/trainer all work from those refs. A
+`data:` URL reaching a renderer ("requires offloaded file:// image assets")
+means ingress was bypassed.
+
+- The image directory comes from `[multimodal].offload_dir` in the resolved
+  config; unset, it defaults to a run-scoped path (`{output_dir}/assets/images`
+  or the hosted `RUN_ID` path). Each env-server resolves that into
+  `VF_RENDERER_IMAGE_OFFLOAD_DIR` for verifiers/renderers (protected — set via
+  `[multimodal].offload_dir`, not `env_vars`).
+- While multimodal rollouts are in flight, image files should accumulate under
+  that directory. Zero files means the offload path isn't being exercised or
+  image preparation failed before request submission.
+- Inference rejects bad refs with `invalid_mm_image_ref` 400s (hash mismatch,
+  fingerprint mismatch, unreadable asset) — grep the inference log.
+- Inference caches materialized refs and logs
+  `mm materialize cache: hits=X misses=Y hit_rate=Z% bytes=A/B evictions=C`
+  every 1000 lookups. Hit rate should climb after turn 1 of multi-turn
+  multimodal rollouts; a stuck-at-zero hit rate with repeat images means the
+  cache is disabled or thrashing (sized by `PRIME_RL_MM_MATERIALIZE_CACHE_GB`,
+  default 2.0, `0` disables).
+- The orchestrator raises on placeholder/token drift ("does not cover
+  image-typed tokens") before a sample ships — treat any occurrence as a bug,
+  not noise.
+- Trainer metrics: `mm/images_materialized` and `time/mm_materialize`. Missing
+  image files fail the trainer hard ("raw image materialization failed") —
+  check whether something cleaned the offload directory mid-run.
+
 ### Process tree
 
 All processes use `setproctitle` so they're visible in `ps`/`htop`/`pstree`:

src/prime_rl/entrypoints/env_server.py

@@ -1,3 +1,4 @@
+import os
 from functools import partial
 
 from verifiers.v1 import pool_serve_kwargs
@@ -7,11 +8,17 @@
 from prime_rl.orchestrator.utils import setup_env_server_logging
 from prime_rl.utils.config import cli
 from prime_rl.utils.process import set_proc_title
+from prime_rl.utils.run_assets import IMAGE_OFFLOAD_DIR_ENV, resolve_image_offload_dir
 from prime_rl.utils.utils import clean_exit
 
 
 @clean_exit
 def run_server(config: EnvServerConfig):
+    # Renderers offload images to the dir named by this env var; resolve it from
+    # this server's own config, letting an operator-set env win (multi-node).
+    os.environ.setdefault(
+        IMAGE_OFFLOAD_DIR_ENV, str(resolve_image_offload_dir(config.output_dir, config.multimodal, os.environ))
+    )
     # ``serve.pool`` (static or elastic) sizes the server; a v0/legacy env runs through
     # the bridge, a v1 env is a native env block — both speak the same serve protocol,
     # so the orchestrator is agnostic. serve_env applies the logging setup in this process

src/prime_rl/entrypoints/rl.py

@@ -105,6 +105,8 @@ def write_subconfigs(config: RLConfig, output_dir: Path) -> None:
             "serve": {**source_dict.get("serve", {}), "address": address},
             "legacy": source_dict.get("legacy", {}),
             "log": {"level": config.orchestrator.log.vf_level, "json_logging": config.orchestrator.log.json_logging},
+            "output_dir": str(config.orchestrator.output_dir),
+            "multimodal": to_toml_dict(config.multimodal),
         }
         with open(env_dir / f"{source.resolved_name}.toml", "wb") as f:
             tomli_w.dump(env_server_dict, f)
@@ -157,6 +159,7 @@ def rl_local(config: RLConfig):
         "WANDB_SHARED_MODE": "1",
         "WANDB_SHARED_RUN_ID": os.environ.get("WANDB_SHARED_RUN_ID", uuid.uuid4().hex),
     }
+    inherited_env = dict(os.environ)
 
     # Validate client port matches inference server port
     if config.inference is not None and not config.orchestrator.model.client.is_elastic:
@@ -200,7 +203,7 @@ def sigterm_handler(signum, frame):
                 inference_process = Popen(
                     inference_cmd,
                     env={
-                        **os.environ,
+                        **inherited_env,
                         **DEFAULT_COMMON_ENV_VARS,
                         **DEFAULT_INFERENCE_ENV_VARS,
                         **config.env_vars,
@@ -290,7 +293,7 @@ def sigterm_handler(signum, frame):
                 stdout=log_file,
                 stderr=log_file,
                 env={
-                    **os.environ,
+                    **inherited_env,
                     **DEFAULT_COMMON_ENV_VARS,
                     "LOGURU_FORCE_COLORS": "1",
                     "WANDB_PROGRAM": "uv run rl",
@@ -339,7 +342,7 @@ def sigterm_handler(signum, frame):
             trainer_process = Popen(
                 trainer_cmd,
                 env={
-                    **os.environ,
+                    **inherited_env,
                     **DEFAULT_COMMON_ENV_VARS,
                     **DEFAULT_TRAINER_ENV_VARS,
                     "LOGURU_FORCE_COLORS": "1",
@@ -437,6 +440,9 @@ def write_slurm_script(config: RLConfig, config_dir: Path, script_path: Path) ->
         kv_offload_disk_path=str(offload.disk.path) if (is_mooncake and offload.disk is not None) else "",
         kv_offload_device_name=offload.device_name if is_mooncake else "",
     )
+    image_offload_dir = (
+        os.path.expanduser(str(config.multimodal.offload_dir)) if config.multimodal.offload_dir is not None else ""
+    )
 
     # Per-component env vars: launcher defaults (shared + multi-node-specific) with the
     # user's config merged on top. Runtime wiring stays in the template.
@@ -463,6 +469,7 @@ def write_slurm_script(config: RLConfig, config_dir: Path, script_path: Path) ->
             **config.slurm.template_vars,
             config_path=config_dir / RL_TOML,
             output_dir=config.output_dir,
+            image_offload_dir=image_offload_dir,
             gpus_per_node=config.deployment.gpus_per_node,
         )
     elif config.inference is not None and config.inference.deployment.type == "disaggregated":
@@ -474,6 +481,7 @@ def write_slurm_script(config: RLConfig, config_dir: Path, script_path: Path) ->
             config_dir=config_dir,
             output_dir=config.output_dir,
             orchestrator_output_dir=config.orchestrator.output_dir,
+            image_offload_dir=image_offload_dir,
             num_train_nodes=config.deployment.num_train_nodes,
             num_infer_nodes=infer_deploy.num_nodes * config.deployment.num_infer_replicas,
             nodes_per_infer_replica=infer_deploy.num_nodes,
@@ -515,6 +523,7 @@ def write_slurm_script(config: RLConfig, config_dir: Path, script_path: Path) ->
             config_dir=config_dir,  # TODO: should prob have each subconfig path separately
             output_dir=config.output_dir,
             orchestrator_output_dir=config.orchestrator.output_dir,
+            image_offload_dir=image_offload_dir,
             num_train_nodes=config.deployment.num_train_nodes,
             num_infer_nodes=config.deployment.total_infer_nodes,
             nodes_per_infer_replica=config.deployment.infer_nodes_per_replica,