diff --git a/autotest/config-npu.yaml b/autotest/config-npu.yaml index 825284b54b..ef86a821d1 100644 --- a/autotest/config-npu.yaml +++ b/autotest/config-npu.yaml @@ -26,7 +26,7 @@ default_config: - HF_HUB_OFFLINE=1 case: - npu-qwen3-sft: + qwen3: - type: sft parameters: @@ -53,7 +53,7 @@ case: runtime_info/text_tokens: 0 timeout: 10800 - npu-qwen3-sft-ep8: + qwen3-ep8: - type: sft phase: first @@ -107,7 +107,7 @@ case: runtime_info/text_tokens: 0 timeout: 10800 - npu-qwen3-sft-tp2: + qwen3-tp2: - type: sft parameters: @@ -133,7 +133,7 @@ case: runtime_info/text_tokens: 0 timeout: 10800 - npu-qwen3-sft-recompute: + qwen3-recompute: - type: sft parameters: @@ -161,7 +161,7 @@ case: runtime_info/text_tokens: 0 timeout: 10800 - npu-qwen3-sft-16nums: + qwen3-16nums: - type: sft parameters: @@ -184,7 +184,7 @@ case: lr: 0 timeout: 10800 - npu-qwen3-sft-celoss-vs-gpu: + qwen3-celoss: - type: sft parameters: @@ -207,7 +207,7 @@ case: lr: 0.01 timeout: 10800 - npu-qwen3-rl-vllm: + qwen3-vllm: - type: rl parameters: diff --git a/autotest/config.yaml b/autotest/config.yaml index db554b7a55..bf6607dd7d 100644 --- a/autotest/config.yaml +++ b/autotest/config.yaml @@ -26,6 +26,268 @@ default_config: - HF_HUB_OFFLINE=1 case: + + glm5-2-rl-lmdeploy-grpo: + - + type: rl + parameters: + config: autotest/config/rl_glm5p2_30B_gsm8k_grpo.py + infer_backend: lmdeploy + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + pip_package: FLASH_MLA_DISABLE_SM100=1 pip install -v /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/flash-mla;pip install -e .[all]; pip install more-itertools pytest-xdist + memory_per_task: 1200 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/gsm8k/train-mini.jsonl + - EVAL_DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/gsm8k/test.jsonl + - XTUNER_USE_LMDEPLOY=1 + - XTUNER_DETERMINISTIC=true + assert_info: + base_metric: glm5-2-rl-lmdeploy-grpo/tracker.jsonl + check_metrics: + - + metric: eval/accuracy + threshold: 0.1 + method: absolute + operator: < + - + metric: response/rewards/mean + threshold: 0.3 + method: absolute + operator: < + - + metric: mismatch/mismatch_k3_kl + threshold: 0.001 + method: absolute + operator: < + - + metric: response/response_len/mean + threshold: 0.25 + method: relative + operator: < + - + metric: time/step + threshold: 20 + method: absolute + operator: < + timeout: 7200 + + glm5-2-rl-lmdeploy-dapo: + - + type: rl + parameters: + config: autotest/config/rl_glm5p2_30B_gsm8k_dapo.py + infer_backend: lmdeploy + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + memory_per_task: 1200 + pip_package: FLASH_MLA_DISABLE_SM100=1 pip install -v /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/flash-mla;pip install -e .[all]; pip install more-itertools pytest-xdist + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/gsm8k/train-mini.jsonl + - EVAL_DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/gsm8k/test.jsonl + - XTUNER_USE_LMDEPLOY=1 + - XTUNER_DETERMINISTIC=true + assert_info: + base_metric: glm5-2-rl-lmdeploy-dapo/tracker.jsonl + check_metrics: + - + metric: eval/accuracy + threshold: 0.1 + method: absolute + operator: < + - + metric: response/rewards/mean + threshold: 0.3 + method: absolute + operator: < + - + metric: mismatch/mismatch_k3_kl + threshold: 0.001 + method: absolute + operator: < + - + metric: response/response_len/mean + threshold: 0.25 + method: relative + operator: < + - + metric: time/step + threshold: 20 + method: absolute + operator: < + timeout: 7200 + glm5-2-sft-30B: + - + type: sft + parameters: + config: autotest/config/glm5p2_30B.py + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + cpus_per_task: 120 + memory_per_task: 1200 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - ALPACA_PATH=/mnt/shared-storage-user/llmrazor-share/data/alpaca + - XTUNER_GC_ENABLE=1 + - SWAP_OPTIMIZER=0 + - XTUNER_ACTIVATION_OFFLOAD=0 + - XTUNER_USE_CUTLASS_GROUP_GEMM=1 + - XTUNER_DETERMINISTIC=true + assert_info: + base_metric: glm5-2-sft-30B/tracker.jsonl + check_metrics: + grad_norm: 0.000001 + loss/local_loss: 0.000001 + loss/reduced_balancing_loss: 0.000001 + loss/reduced_llm_loss: 0.000001 + lr: 0 + memory/max_memory_GB: 0.2 + runtime_info/tgs: 0.05 + runtime_info/text_tokens: 0 + timeout: 1500 + + glm5-2-sft-30B-mtp-fp8: + - + type: sft + parameters: + config: autotest/config/glm5p2_30B_mtp_fp8.py + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + cpus_per_task: 120 + memory_per_task: 1200 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - ALPACA_PATH=/mnt/shared-storage-user/llmrazor-share/data/alpaca + - XTUNER_DETERMINISTIC=true + - XTUNER_ACTIVATION_OFFLOAD=1 + assert_info: + base_metric: glm5-2-sft-30B-mtp-fp8/tracker.jsonl + check_metrics: + grad_norm: 0.1 + loss/local_loss: 0.000001 + loss/reduced_balancing_loss: 0.000001 + loss/reduced_llm_loss: 0.000001 + lr: 0 + memory/max_memory_GB: 0.2 + runtime_info/tgs: 0.05 + runtime_info/text_tokens: 0 + timeout: 1500 + + glm5-2-sft-30B-sp2: + - + type: sft + parameters: + config: autotest/config/glm5p2_30B_sp2.py + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + cpus_per_task: 120 + memory_per_task: 1200 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - ALPACA_PATH=/mnt/shared-storage-user/llmrazor-share/data/alpaca + - XTUNER_DETERMINISTIC=true + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + assert_info: + base_metric: glm5-2-sft-30B-sp2/tracker.jsonl + check_metrics: + grad_norm: 0.000001 + loss/local_loss: 0.000001 + loss/reduced_balancing_loss: 0.000001 + loss/reduced_llm_loss: 0.000001 + lr: 0 + memory/max_memory_GB: 0.2 + runtime_info/tgs: 0.05 + runtime_info/text_tokens: 0 + timeout: 1500 + + glm5-2-sft-30B-ep2-resume: + - + type: sft + phase: first + parameters: + config: autotest/config/glm5p2_30B_ep2_resume.py + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + num_nodes: 2 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - ALPACA_PATH=/mnt/shared-storage-user/llmrazor-share/data/alpaca + - XTUNER_DETERMINISTIC=true + - XTUNER_GC_ENABLE=1 + assert_info: + base_metric: glm5-2-sft-30B-ep2-resume/tracker.jsonl + check_metrics: + grad_norm: 0.000001 + loss/local_loss: 0.000001 + loss/reduced_balancing_loss: 0.000001 + loss/reduced_llm_loss: 0.000001 + lr: 0 + memory/max_memory_GB: 0.2 + runtime_info/tgs: 0.05 + runtime_info/text_tokens: 0 + timeout: 1500 + + - + type: sft + phase: resume + pre_action: + command: 'python ./autotest/utils/update_meta.py /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output glm5-2-sft-30B-ep2-resume sft' + parameters: + config: autotest/config/glm5p2_30B_ep2_resume.py + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + num_nodes: 2 + cpus_per_task: 80 + memory_per_task: 1200 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - ALPACA_PATH=/mnt/shared-storage-user/llmrazor-share/data/alpaca + - XTUNER_DETERMINISTIC=true + - XTUNER_GC_ENABLE=1 + assert_info: + base_metric: glm5-2-sft-30B-ep2-resume/tracker-resume.jsonl + check_metrics: + grad_norm: 0.000001 + loss/local_loss: 0.000001 + loss/reduced_balancing_loss: 0.000001 + loss/reduced_llm_loss: 0.000001 + lr: 0 + memory/max_memory_GB: 0.2 + runtime_info/tgs: 0.05 + runtime_info/text_tokens: 0 + post_action: + command: 'python ./autotest/utils/resume_validation.py /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output glm5-2-sft-30B-ep2-resume sft 10 "runtime_info/text_tokens,runtime_info/efficient_attn_ratio,loss/reduced_balancing_loss,loss/reduced_llm_loss,loss/local_loss"' + timeout: 1500 + + glm5-2-sft-30B-ep8-sp2-tp2: + - + type: sft + parameters: + config: autotest/config/glm5p2_30B_ep8_sp2_tp2.py + output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output + resource: + num_nodes: 2 + cpus_per_task: 80 + envs: + - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/GLM-5.2-30B + - ALPACA_PATH=/mnt/shared-storage-user/llmrazor-share/data/alpaca + - XTUNER_DETERMINISTIC=true + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + assert_info: + base_metric: glm5-2-sft-30B-ep8-sp2-tp2/tracker.jsonl + check_metrics: + grad_norm: 0.000001 + loss/local_loss: 0.000001 + loss/reduced_balancing_loss: 0.000001 + loss/reduced_llm_loss: 0.000001 + lr: 0 + memory/max_memory_GB: 0.2 + runtime_info/tgs: 0.05 + runtime_info/text_tokens: 0 + timeout: 1800 + # InternS2-Preview: Qwen3.5-VL-MoE-35B-A3B architecture; MODEL_PATH may need adjust on share. s2-preview-rl-vl-lmdeploy-grpo: - diff --git a/autotest/config/glm5p2_30B.py b/autotest/config/glm5p2_30B.py new file mode 100644 index 0000000000..ba2dbf94e7 --- /dev/null +++ b/autotest/config/glm5p2_30B.py @@ -0,0 +1,63 @@ +import os + +from xtuner.v1.config import ( + AdamWConfig, + FSDPConfig, + LRConfig, +) +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.sft_tokenize_fn import OpenaiTokenizeFunctionConfig +from xtuner.v1.loss.ce_loss import CELossConfig +from xtuner.v1.model import get_model_config_from_hf +from xtuner.v1.train import TrainerConfig + + +MODEL_PATH = os.environ["MODEL_PATH"] +ALPACA_PATH = os.environ["ALPACA_PATH"] + +ep_size = 4 + +moe_cfg = get_model_config_from_hf(MODEL_PATH) +moe_cfg.dispatcher = "deepep" +moe_cfg.ep_size = ep_size +moe_cfg.compile_cfg = False +if hasattr(moe_cfg.attention, "sparse_mla_backend"): + moe_cfg.attention.sparse_mla_backend = "tilelang" + +optim_cfg = AdamWConfig(lr=6e-05) +lr_cfg = LRConfig(lr_type="cosine", lr_min=1e-6) +fsdp_cfg = FSDPConfig( + cpu_offload=False, + ep_size=ep_size, +) + +dataset_config = [ + { + "dataset": DatasetConfig(name="alpaca", anno_path=ALPACA_PATH, sample_ratio=1.0), + "tokenize_fn": OpenaiTokenizeFunctionConfig(chat_template="glm5.2", max_length=4096), + }, +] + +dataloader_config = DataloaderConfig(pack_max_length=16384) + +loss_cfg = CELossConfig(mode="chunk", chunk_size=1024) +moe_cfg.lm_loss_cfg = loss_cfg + +trainer = TrainerConfig( + load_from=MODEL_PATH, + model_cfg=moe_cfg, + optim_cfg=optim_cfg, + fsdp_cfg=fsdp_cfg, + dataset_cfg=dataset_config, + dataloader_cfg=dataloader_config, + lr_cfg=lr_cfg, + loss_cfg=loss_cfg, + tokenizer_path=MODEL_PATH, + strict_load=True, + global_batch_size=8, + intra_layer_micro_batch=1, + sp_size=1, + total_epoch=1, + work_dir=f"{os.environ['WORK_DIR']}", + seed=0, +) diff --git a/autotest/config/glm5p2_30B_ep2_resume.py b/autotest/config/glm5p2_30B_ep2_resume.py new file mode 100644 index 0000000000..552e464e1d --- /dev/null +++ b/autotest/config/glm5p2_30B_ep2_resume.py @@ -0,0 +1,66 @@ +import os + +from xtuner.v1.config import ( + AdamWConfig, + FSDPConfig, + LRConfig, +) +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.sft_tokenize_fn import OpenaiTokenizeFunctionConfig +from xtuner.v1.loss.ce_loss import CELossConfig +from xtuner.v1.model import get_model_config_from_hf +from xtuner.v1.train import ResumeConfig, TrainerConfig + + +MODEL_PATH = os.environ["MODEL_PATH"] +ALPACA_PATH = os.environ["ALPACA_PATH"] + +ep_size = 2 + +moe_cfg = get_model_config_from_hf(MODEL_PATH) +moe_cfg.dispatcher = "deepep" +moe_cfg.ep_size = ep_size +moe_cfg.compile_cfg = False +if hasattr(moe_cfg.attention, "sparse_mla_backend"): + moe_cfg.attention.sparse_mla_backend = "tilelang" + +optim_cfg = AdamWConfig(lr=6e-05) +lr_cfg = LRConfig(lr_type="cosine", lr_min=1e-6) +fsdp_cfg = FSDPConfig( + cpu_offload=False, + ep_size=ep_size, +) + +dataset_config = [ + { + "dataset": DatasetConfig(name="alpaca", anno_path=ALPACA_PATH, sample_ratio=1.0), + "tokenize_fn": OpenaiTokenizeFunctionConfig(chat_template="glm5.2", max_length=16384), + }, +] + +dataloader_config = DataloaderConfig(pack_max_length=16384) + +loss_cfg = CELossConfig(mode="chunk", chunk_size=1024) +moe_cfg.lm_loss_cfg = loss_cfg + +trainer = TrainerConfig( + load_from=MODEL_PATH, + model_cfg=moe_cfg, + optim_cfg=optim_cfg, + fsdp_cfg=fsdp_cfg, + dataset_cfg=dataset_config, + dataloader_cfg=dataloader_config, + lr_cfg=lr_cfg, + loss_cfg=loss_cfg, + tokenizer_path=MODEL_PATH, + strict_load=True, + global_batch_size=16, + intra_layer_micro_batch=1, + sp_size=1, + total_epoch=1, + work_dir=f"{os.environ['WORK_DIR']}", + seed=0, + resume_cfg=ResumeConfig(auto_resume=True), + checkpoint_interval=10, + checkpoint_maxkeep=2, +) diff --git a/autotest/config/glm5p2_30B_ep8_sp2_tp2.py b/autotest/config/glm5p2_30B_ep8_sp2_tp2.py new file mode 100644 index 0000000000..f3d3bedff5 --- /dev/null +++ b/autotest/config/glm5p2_30B_ep8_sp2_tp2.py @@ -0,0 +1,64 @@ +import os + +from xtuner.v1.config import ( + AdamWConfig, + FSDPConfig, + LRConfig, +) +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.sft_tokenize_fn import OpenaiTokenizeFunctionConfig +from xtuner.v1.loss.ce_loss import CELossConfig +from xtuner.v1.model import get_model_config_from_hf +from xtuner.v1.train import TrainerConfig + + +MODEL_PATH = os.environ["MODEL_PATH"] +ALPACA_PATH = os.environ["ALPACA_PATH"] + +ep_size = 8 + +moe_cfg = get_model_config_from_hf(MODEL_PATH) +moe_cfg.dispatcher = "all2all" +moe_cfg.ep_size = ep_size +moe_cfg.compile_cfg = False +if hasattr(moe_cfg.attention, "sparse_mla_backend"): + moe_cfg.attention.sparse_mla_backend = "tilelang" + +optim_cfg = AdamWConfig(lr=6e-05) +lr_cfg = LRConfig(lr_type="cosine", lr_min=1e-6) +fsdp_cfg = FSDPConfig( + cpu_offload=False, + ep_size=ep_size, + tp_size=2, +) + +dataset_config = [ + { + "dataset": DatasetConfig(name="alpaca", anno_path=ALPACA_PATH, sample_ratio=1.0), + "tokenize_fn": OpenaiTokenizeFunctionConfig(chat_template="glm5.2", max_length=16384), + }, +] + +dataloader_config = DataloaderConfig(pack_max_length=16384) + +loss_cfg = CELossConfig(mode="chunk", chunk_size=1024) +moe_cfg.lm_loss_cfg = loss_cfg + +trainer = TrainerConfig( + load_from=MODEL_PATH, + model_cfg=moe_cfg, + optim_cfg=optim_cfg, + fsdp_cfg=fsdp_cfg, + dataset_cfg=dataset_config, + dataloader_cfg=dataloader_config, + lr_cfg=lr_cfg, + loss_cfg=loss_cfg, + tokenizer_path=MODEL_PATH, + strict_load=True, + global_batch_size=8, + intra_layer_micro_batch=2, + sp_size=2, + total_step=20, + work_dir=f"{os.environ['WORK_DIR']}", + seed=0, +) diff --git a/autotest/config/glm5p2_30B_mtp_fp8.py b/autotest/config/glm5p2_30B_mtp_fp8.py new file mode 100644 index 0000000000..79e01207dc --- /dev/null +++ b/autotest/config/glm5p2_30B_mtp_fp8.py @@ -0,0 +1,75 @@ +import os + +from xtuner.v1.config import ( + AdamWConfig, + FSDPConfig, + LRConfig, +) +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.sft_tokenize_fn import OpenaiTokenizeFunctionConfig +from xtuner.v1.float8.config import Float8Config, ScalingGranularity +from xtuner.v1.loss.ce_loss import CELossConfig +from xtuner.v1.model import get_model_config_from_hf +from xtuner.v1.module.mtp import MTPConfig +from xtuner.v1.train import TrainerConfig + + +MODEL_PATH = os.environ["MODEL_PATH"] +ALPACA_PATH = os.environ["ALPACA_PATH"] + +ep_size = 1 + +float8_cfg = Float8Config( + scaling_granularity_gemm=ScalingGranularity.TILEWISE, + scaling_granularity_grouped_gemm=ScalingGranularity.TILEWISE, +) + +moe_cfg = get_model_config_from_hf(MODEL_PATH) +moe_cfg.dispatcher = "all2all" +moe_cfg.ep_size = ep_size +moe_cfg.compile_cfg = False +moe_cfg.float8_cfg = float8_cfg +moe_cfg.mtp_config = MTPConfig(num_layers=4, share_weights=True) +moe_cfg.num_nextn_predict_layers = 4 +if hasattr(moe_cfg, "_normalize_physical_mtp_indexer_types"): + moe_cfg._normalize_physical_mtp_indexer_types() +if hasattr(moe_cfg.attention, "sparse_mla_backend"): + moe_cfg.attention.sparse_mla_backend = "tilelang" + +optim_cfg = AdamWConfig(lr=6e-05) +lr_cfg = LRConfig(lr_type="cosine", lr_min=1e-6) +fsdp_cfg = FSDPConfig( + cpu_offload=False, + ep_size=ep_size, +) + +dataset_config = [ + { + "dataset": DatasetConfig(name="alpaca", anno_path=ALPACA_PATH, sample_ratio=1.0), + "tokenize_fn": OpenaiTokenizeFunctionConfig(chat_template="glm5.2", max_length=16384), + }, +] + +dataloader_config = DataloaderConfig(pack_max_length=16384) + +loss_cfg = CELossConfig(mode="chunk", chunk_size=1024, loss_reduction="square") +moe_cfg.lm_loss_cfg = loss_cfg + +trainer = TrainerConfig( + load_from=MODEL_PATH, + model_cfg=moe_cfg, + optim_cfg=optim_cfg, + fsdp_cfg=fsdp_cfg, + dataset_cfg=dataset_config, + dataloader_cfg=dataloader_config, + lr_cfg=lr_cfg, + loss_cfg=loss_cfg, + tokenizer_path=MODEL_PATH, + strict_load=True, + global_batch_size=8, + intra_layer_micro_batch=1, + sp_size=1, + total_epoch=1, + work_dir=f"{os.environ['WORK_DIR']}", + seed=0, +) diff --git a/autotest/config/glm5p2_30B_sp2.py b/autotest/config/glm5p2_30B_sp2.py new file mode 100644 index 0000000000..3558750e0d --- /dev/null +++ b/autotest/config/glm5p2_30B_sp2.py @@ -0,0 +1,64 @@ +import os + +from xtuner.v1.config import ( + AdamWConfig, + FSDPConfig, + LRConfig, +) +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.sft_tokenize_fn import OpenaiTokenizeFunctionConfig +from xtuner.v1.loss.ce_loss import CELossConfig +from xtuner.v1.model import get_model_config_from_hf +from xtuner.v1.train import TrainerConfig + + +MODEL_PATH = os.environ["MODEL_PATH"] +ALPACA_PATH = os.environ["ALPACA_PATH"] + +ep_size = 2 + +moe_cfg = get_model_config_from_hf(MODEL_PATH) +moe_cfg.dispatcher = "all2all" +moe_cfg.ep_size = ep_size +moe_cfg.compile_cfg = False +if hasattr(moe_cfg.attention, "sparse_mla_backend"): + moe_cfg.attention.sparse_mla_backend = "tilelang" + +optim_cfg = AdamWConfig(lr=6e-05) +lr_cfg = LRConfig(lr_type="cosine", lr_min=1e-6) +fsdp_cfg = FSDPConfig( + cpu_offload=False, + ep_size=ep_size, + tp_size=1, +) + +dataset_config = [ + { + "dataset": DatasetConfig(name="alpaca", anno_path=ALPACA_PATH, sample_ratio=1.0), + "tokenize_fn": OpenaiTokenizeFunctionConfig(chat_template="glm5.2", max_length=16384), + }, +] + +dataloader_config = DataloaderConfig(pack_max_length=16384) + +loss_cfg = CELossConfig(mode="chunk", chunk_size=1024) +moe_cfg.lm_loss_cfg = loss_cfg + +trainer = TrainerConfig( + load_from=MODEL_PATH, + model_cfg=moe_cfg, + optim_cfg=optim_cfg, + fsdp_cfg=fsdp_cfg, + dataset_cfg=dataset_config, + dataloader_cfg=dataloader_config, + lr_cfg=lr_cfg, + loss_cfg=loss_cfg, + tokenizer_path=MODEL_PATH, + strict_load=True, + global_batch_size=8, + intra_layer_micro_batch=2, + sp_size=2, + total_step=20, + work_dir=f"{os.environ['WORK_DIR']}", + seed=0, +) diff --git a/autotest/config/rl_glm5p2_30B_gsm8k_dapo.py b/autotest/config/rl_glm5p2_30B_gsm8k_dapo.py new file mode 100644 index 0000000000..465f3c6b13 --- /dev/null +++ b/autotest/config/rl_glm5p2_30B_gsm8k_dapo.py @@ -0,0 +1,214 @@ +import os +from pathlib import Path + +from xtuner.v1.config import AdamWConfig, FSDPConfig, LRConfig +from xtuner.v1.data_proto.rl_data import SampleParams +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.rl_tokenize_fn import RLTextTokenizeFnConfig +from xtuner.v1.model import get_model_config_from_hf +from xtuner.v1.rl.advantage import GRPOAdvantageConfig +from xtuner.v1.rl.agent_loop import SingleTurnAgentLoopConfig +from xtuner.v1.rl.agent_loop_manager import ( + AgentLoopManagerConfig, + SamplerConfig, + SyncProduceStrategyConfig, + TaskSpecConfig, +) +from xtuner.v1.rl.evaluator import EvaluatorConfig +from xtuner.v1.rl.judger import GSM8KJudgerConfig +from xtuner.v1.rl.loss import GRPOLossConfig +from xtuner.v1.rl.replay_buffer import SyncReplayBufferConfig +from xtuner.v1.rl.rollout.worker import RolloutConfig +from xtuner.v1.rl.trainer import WorkerConfig +from xtuner.v1.rl.utils import AcceleratorResourcesConfig, CPUResourcesConfig +from xtuner.v1.train.rl_trainer import RLColocateTrainerConfig + + +# env +work_dir = os.environ["WORK_DIR"] +model_path = os.environ["MODEL_PATH"] +data_path = os.environ["DATA_PATH"] +eval_data_path = os.environ["EVAL_DATA_PATH"] +enable_return_routed_experts = os.environ.get("ENABLE_RETURN_ROUTED_EXPERTS", "1") +NNODE = int(os.environ.get("WORLD_SIZE", "1")) + +# basic settings +experimental_name = "glm5p2_dapo_gsm8k" +total_train_steps = 15 +evaluate_step = 15 +train_optimizer_steps = 1 +train_batch_size = 64 * train_optimizer_steps +prompt_repeat_k = 5 +rollout_tp_size = 1 +rollout_ep_size = 8 +max_prompt_length = 512 +max_response_length = 1024 +pack_max_length = 16 * 1024 + +# 1. resources +resources = AcceleratorResourcesConfig( + accelerator="GPU", + num_workers=8 * NNODE, + num_cpus_per_worker=12, + cpu_memory_per_worker=16 * 1024**3, # 16 GB +) + +# 2. rollout +rollout_config = RolloutConfig( + env=experimental_name, + device=resources.accelerator, + model_path=model_path, + dtype="bfloat16", + tensor_parallel_size=rollout_tp_size, + expert_parallel_size=rollout_ep_size, + gpu_memory_utilization=0.8, + context_length=max_response_length + max_prompt_length, + enable_return_routed_experts=(enable_return_routed_experts == "1"), + extra_rollout_config=dict( + lmdeploy_trust_remote_code=True, + lmdeploy_log_level="INFO", + lmdeploy_uvicorn_log_level="INFO", + ), +) + +# 3. judger +judger_config = GSM8KJudgerConfig( + judger_name="openai/gsm8k", + cpu_resources=CPUResourcesConfig(num_workers=1, num_cpus_per_worker=1), +) + +# 4. train worker +lr_cfg = LRConfig(lr_type="constant", warmup_ratio=0, lr_min=1e-6) +fsdp_cfg = FSDPConfig(torch_compile=False, cpu_offload=False, ep_size=1) +model_cfg = get_model_config_from_hf(Path(model_path)) +model_cfg.dispatcher = "deepep" +model_cfg.compile_cfg = False +if hasattr(model_cfg, "balancing_loss_cfg"): + model_cfg.balancing_loss_cfg = None +if hasattr(model_cfg, "z_loss_cfg"): + model_cfg.z_loss_cfg = None +if hasattr(model_cfg, "attention") and hasattr(model_cfg.attention, "sparse_mla_backend"): + model_cfg.attention.sparse_mla_backend = "tilelang" +optim_cfg = AdamWConfig(lr=1e-6, foreach=False, weight_decay=0.1) +loss_cfg = GRPOLossConfig( + policy_loss_cfg=dict( + cliprange_high=0.28, + cliprange_low=0.2, + loss_type=os.environ.get("LOSS_TYPE", "vanilla"), + clip_ratio_c=10.0, + log_prob_diff_min=-20.0, + log_prob_diff_max=20.0, + ), + ignore_idx=-100, + use_kl_loss=False, + kl_loss_coef=0.0, + kl_loss_type="low_var_kl", + mode=os.environ.get("LOSS_MODE", "chunk"), + chunk_size=512, +) +train_worker_cfg = WorkerConfig( + model_cfg=model_cfg, + load_from=model_path, + optim_cfg=optim_cfg, + loss_cfg=loss_cfg, + lr_cfg=lr_cfg, + fsdp_cfg=fsdp_cfg, + sp_size=4, + optimizer_steps=train_optimizer_steps, + pack_max_length=pack_max_length, +) + +# 5. train agent loop manager +train_dataset = DatasetConfig(name=experimental_name, anno_path=data_path) +tokenizer_config = RLTextTokenizeFnConfig(max_length=max_prompt_length) +train_dataset_cfg = [{"dataset": train_dataset, "tokenize_fn": tokenizer_config}] +dataloader_cfg = DataloaderConfig( + dataset_config_list=train_dataset_cfg, + pack_max_length=pack_max_length, + collator="fake_collator", + pack_level="none", +) +sampler_config = SamplerConfig( + dataloader_cfg=dataloader_cfg, + prompt_repeat_k=prompt_repeat_k, +) +training_sample_params = SampleParams( + max_tokens=max_response_length, + top_k=0, + top_p=1.0, + temperature=1.0, + min_tokens=0, +) +agent_loop_config = SingleTurnAgentLoopConfig( + hf_checkpoint=model_path, + sample_params=training_sample_params, +) +produce_strategy_config = SyncProduceStrategyConfig() +agent_loop_manager_cfg = AgentLoopManagerConfig( + tasks=TaskSpecConfig( + task_name="train_task", + agent_loop_config=agent_loop_config, + judger_config=judger_config, + produce_strategy_config=produce_strategy_config, + sampler_config=sampler_config, + ), +) + +# 6. eval agent loop manager +eval_dataset = DatasetConfig(name=experimental_name, anno_path=eval_data_path, sample_ratio=1.0) +eval_dataset_cfg = [{"dataset": eval_dataset, "tokenize_fn": tokenizer_config}] +eval_dataloader_cfg = DataloaderConfig( + dataset_config_list=eval_dataset_cfg, + pack_max_length=pack_max_length, + collator="fake_collator", + pack_level="none", +) +eval_sampler_config = SamplerConfig( + dataloader_cfg=eval_dataloader_cfg, + prompt_repeat_k=1, +) +evaluation_sample_params = SampleParams( + max_tokens=max_response_length, + top_k=1, + top_p=1.0, + temperature=0.0, + min_tokens=0, +) +eval_agent_loop_config = SingleTurnAgentLoopConfig( + hf_checkpoint=model_path, + sample_params=evaluation_sample_params, +) +eval_agent_loop_manager_cfg = AgentLoopManagerConfig( + tasks=TaskSpecConfig( + task_name="eval_task", + agent_loop_config=eval_agent_loop_config, + judger_config=judger_config, + sampler_config=eval_sampler_config, + ), +) + +# 7. evaluator +evaluator_config = EvaluatorConfig(compute_metric_func=None) + +# 8. RL Colocate Trainer Config +trainer = RLColocateTrainerConfig( + resources=resources, + train_worker_cfg=train_worker_cfg, + rollout_config=rollout_config, + tokenizer_path=model_path, + replay_buffer_config=SyncReplayBufferConfig(), + agent_loop_manager_cfg=agent_loop_manager_cfg, + eval_agent_loop_manager_cfg=eval_agent_loop_manager_cfg, + evaluator_config=evaluator_config, + load_from=model_path, + total_train_steps=total_train_steps, + train_batch_size=train_batch_size, + advantage_estimator_config=GRPOAdvantageConfig(eps=1e-8), + enable_evaluate=True, + enable_initial_evaluate=False, + evaluate_step=evaluate_step, + work_dir=work_dir, + seed=123, + debug_rollout=False, + exp_tracker="jsonl", +) diff --git a/autotest/config/rl_glm5p2_30B_gsm8k_grpo.py b/autotest/config/rl_glm5p2_30B_gsm8k_grpo.py new file mode 100644 index 0000000000..557e6441f8 --- /dev/null +++ b/autotest/config/rl_glm5p2_30B_gsm8k_grpo.py @@ -0,0 +1,214 @@ +import os +from pathlib import Path + +from xtuner.v1.config import AdamWConfig, FSDPConfig, LRConfig +from xtuner.v1.data_proto.rl_data import SampleParams +from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig +from xtuner.v1.datasets.rl_tokenize_fn import RLTextTokenizeFnConfig +from xtuner.v1.model import get_model_config_from_hf +from xtuner.v1.rl.advantage import GRPOAdvantageConfig +from xtuner.v1.rl.agent_loop import SingleTurnAgentLoopConfig +from xtuner.v1.rl.agent_loop_manager import ( + AgentLoopManagerConfig, + SamplerConfig, + SyncProduceStrategyConfig, + TaskSpecConfig, +) +from xtuner.v1.rl.evaluator import EvaluatorConfig +from xtuner.v1.rl.judger import GSM8KJudgerConfig +from xtuner.v1.rl.loss import GRPOLossConfig +from xtuner.v1.rl.replay_buffer import SyncReplayBufferConfig +from xtuner.v1.rl.rollout.worker import RolloutConfig +from xtuner.v1.rl.trainer import WorkerConfig +from xtuner.v1.rl.utils import AcceleratorResourcesConfig, CPUResourcesConfig +from xtuner.v1.train.rl_trainer import RLColocateTrainerConfig + + +# env +work_dir = os.environ["WORK_DIR"] +model_path = os.environ["MODEL_PATH"] +data_path = os.environ["DATA_PATH"] +eval_data_path = os.environ["EVAL_DATA_PATH"] +enable_return_routed_experts = os.environ.get("ENABLE_RETURN_ROUTED_EXPERTS", "1") +NNODE = int(os.environ.get("WORLD_SIZE", "1")) + +# basic settings +experimental_name = "glm5p2_dapo_gsm8k" +total_train_steps = 15 +evaluate_step = 15 +train_optimizer_steps = 1 +train_batch_size = 16 * train_optimizer_steps +prompt_repeat_k = 5 +rollout_tp_size = 1 +rollout_ep_size = 8 +max_prompt_length = 512 +max_response_length = 1024 +pack_max_length = 8192 + +# 1. resources +resources = AcceleratorResourcesConfig( + accelerator="GPU", + num_workers=8 * NNODE, + num_cpus_per_worker=12, + cpu_memory_per_worker=16 * 1024**3, # 16 GB +) + +# 2. rollout +rollout_config = RolloutConfig( + env=experimental_name, + device=resources.accelerator, + model_path=model_path, + dtype="bfloat16", + tensor_parallel_size=rollout_tp_size, + expert_parallel_size=rollout_ep_size, + gpu_memory_utilization=0.8, + context_length=max_response_length + max_prompt_length, + enable_return_routed_experts=(enable_return_routed_experts == "1"), + extra_rollout_config=dict( + lmdeploy_trust_remote_code=True, + lmdeploy_log_level="INFO", + lmdeploy_uvicorn_log_level="INFO", + ), +) + +# 3. judger +judger_config = GSM8KJudgerConfig( + judger_name="openai/gsm8k", + cpu_resources=CPUResourcesConfig(num_workers=1, num_cpus_per_worker=1), +) + +# 4. train worker +lr_cfg = LRConfig(lr_type="constant", warmup_ratio=0, lr_min=1e-6) +fsdp_cfg = FSDPConfig(torch_compile=False, cpu_offload=False, ep_size=1) +model_cfg = get_model_config_from_hf(Path(model_path)) +model_cfg.dispatcher = "all2all" +model_cfg.compile_cfg = False +if hasattr(model_cfg, "balancing_loss_cfg"): + model_cfg.balancing_loss_cfg = None +if hasattr(model_cfg, "z_loss_cfg"): + model_cfg.z_loss_cfg = None +if hasattr(model_cfg, "attention") and hasattr(model_cfg.attention, "sparse_mla_backend"): + model_cfg.attention.sparse_mla_backend = "tilelang" +optim_cfg = AdamWConfig(lr=1e-6, foreach=False, weight_decay=0.1) +loss_cfg = GRPOLossConfig( + policy_loss_cfg=dict( + cliprange_high=0.28, + cliprange_low=0.2, + loss_type=os.environ.get("LOSS_TYPE", "vanilla"), + clip_ratio_c=10.0, + log_prob_diff_min=-20.0, + log_prob_diff_max=20.0, + ), + ignore_idx=-100, + use_kl_loss=False, + kl_loss_coef=0.0, + kl_loss_type="low_var_kl", + mode=os.environ.get("LOSS_MODE", "chunk"), + chunk_size=512, +) +train_worker_cfg = WorkerConfig( + model_cfg=model_cfg, + load_from=model_path, + optim_cfg=optim_cfg, + loss_cfg=loss_cfg, + lr_cfg=lr_cfg, + fsdp_cfg=fsdp_cfg, + sp_size=2, + optimizer_steps=train_optimizer_steps, + pack_max_length=pack_max_length, +) + +# 5. train agent loop manager +train_dataset = DatasetConfig(name=experimental_name, anno_path=data_path) +tokenizer_config = RLTextTokenizeFnConfig(max_length=max_prompt_length) +train_dataset_cfg = [{"dataset": train_dataset, "tokenize_fn": tokenizer_config}] +dataloader_cfg = DataloaderConfig( + dataset_config_list=train_dataset_cfg, + pack_max_length=pack_max_length, + collator="fake_collator", + pack_level="none", +) +sampler_config = SamplerConfig( + dataloader_cfg=dataloader_cfg, + prompt_repeat_k=prompt_repeat_k, +) +training_sample_params = SampleParams( + max_tokens=max_response_length, + top_k=0, + top_p=1.0, + temperature=1.0, + min_tokens=0, +) +agent_loop_config = SingleTurnAgentLoopConfig( + hf_checkpoint=model_path, + sample_params=training_sample_params, +) +produce_strategy_config = SyncProduceStrategyConfig() +agent_loop_manager_cfg = AgentLoopManagerConfig( + tasks=TaskSpecConfig( + task_name="train_task", + agent_loop_config=agent_loop_config, + judger_config=judger_config, + produce_strategy_config=produce_strategy_config, + sampler_config=sampler_config, + ), +) + +# 6. eval agent loop manager +eval_dataset = DatasetConfig(name=experimental_name, anno_path=eval_data_path, sample_ratio=1.0) +eval_dataset_cfg = [{"dataset": eval_dataset, "tokenize_fn": tokenizer_config}] +eval_dataloader_cfg = DataloaderConfig( + dataset_config_list=eval_dataset_cfg, + pack_max_length=pack_max_length, + collator="fake_collator", + pack_level="none", +) +eval_sampler_config = SamplerConfig( + dataloader_cfg=eval_dataloader_cfg, + prompt_repeat_k=1, +) +evaluation_sample_params = SampleParams( + max_tokens=max_response_length, + top_k=1, + top_p=1.0, + temperature=0.0, + min_tokens=0, +) +eval_agent_loop_config = SingleTurnAgentLoopConfig( + hf_checkpoint=model_path, + sample_params=evaluation_sample_params, +) +eval_agent_loop_manager_cfg = AgentLoopManagerConfig( + tasks=TaskSpecConfig( + task_name="eval_task", + agent_loop_config=eval_agent_loop_config, + judger_config=judger_config, + sampler_config=eval_sampler_config, + ), +) + +# 7. evaluator +evaluator_config = EvaluatorConfig(compute_metric_func=None) + +# 8. RL Colocate Trainer Config +trainer = RLColocateTrainerConfig( + resources=resources, + train_worker_cfg=train_worker_cfg, + rollout_config=rollout_config, + tokenizer_path=model_path, + replay_buffer_config=SyncReplayBufferConfig(), + agent_loop_manager_cfg=agent_loop_manager_cfg, + eval_agent_loop_manager_cfg=eval_agent_loop_manager_cfg, + evaluator_config=evaluator_config, + load_from=model_path, + total_train_steps=total_train_steps, + train_batch_size=train_batch_size, + advantage_estimator_config=GRPOAdvantageConfig(eps=1e-8), + enable_evaluate=True, + enable_initial_evaluate=False, + evaluate_step=evaluate_step, + work_dir=work_dir, + seed=123, + debug_rollout=False, + exp_tracker="jsonl", +)