diff --git a/src/prime_rl/trainer/rl/train.py b/src/prime_rl/trainer/rl/train.py index 568ca6e1d8..fe63de8d47 100644 --- a/src/prime_rl/trainer/rl/train.py +++ b/src/prime_rl/trainer/rl/train.py @@ -604,7 +604,12 @@ def train(config: TrainerConfig): broadcast_weights_start_time = time.perf_counter() # The per-layer gather + fp8 conversion peaks ~50 GiB above the # resident weights; release cached blocks (incl. offload-stream - # pools) so the broadcast gets the full headroom. + # pools) so the broadcast gets the full headroom. Drain all + # pending work first: empty_cache returns blocks to the driver, + # so a still-running kernel holding a cached block (e.g. the + # optimizer step's tail) faults with an illegal memory access + # once its block is freed under it. + torch.cuda.synchronize() torch.cuda.empty_cache() weight_sender.broadcast(model, step=progress.step) broadcast_weights_time = time.perf_counter() - broadcast_weights_start_time