test(checkpoint): expand parity metrics and phase coverage

examples/llm_finetune/baichuan/baichuan_2_7b_squad.yaml

@@ -114,11 +114,8 @@ ci:
   known_issue_id: AM-149
   checkpoint_robustness:
     trust_remote_code: true
-    kl_threshold: 1e-2
-    hf_kl_threshold: 5e-2
     distributed.tp_size: 2
     cross_tp_size: 2
-    cross_tp_kl_threshold: 1e-2
     training_reproducibility_loss_threshold: 5e-2
     tokenizer_name: baichuan-inc/Baichuan2-7B-Chat
     dataset.limit_dataset_samples: 500

examples/llm_finetune/baichuan/baichuan_2_7b_squad_peft.yaml

@@ -125,8 +125,6 @@ ci:
   known_issue_id: AM-149
   checkpoint_robustness:
     trust_remote_code: true
-    kl_threshold: 1e-2
-    hf_kl_threshold: 5e-2
     distributed.tp_size: 2
     tokenizer_name: baichuan-inc/Baichuan2-7B-Chat
     dataset.limit_dataset_samples: 500

examples/llm_finetune/deepseek_v4/deepseek_v4_flash_hellaswag_lora.yaml

@@ -130,5 +130,16 @@ optimizer:
 ci:
   # pp_size(1) * ep_size(32) = 32 GPUs => 4 nodes (8 H100/node).
   recipe_owner: hemildesai
-  time: "00:20:00"
+  time: "01:00:00"
   nodes: 4
+  checkpoint_robustness:
+    step_scheduler.global_batch_size: 32
+    step_scheduler.local_batch_size: 1
+    tokenizer_name: deepseek-ai/DeepSeek-V4-Flash
+    trust_remote_code: true
+    hf_device_map_auto: true
+    # EP32 sparse routing is inside the shared relaxed envelope; the separate
+    # vanilla-HF reload gate remains enabled so reference-load regressions still fail.
+    parity_tolerance_profile: relaxed
+    dataset.num_samples_limit: 500
+    validation_dataset.num_samples_limit: 500

examples/llm_finetune/ernie4_5/ernie4_5_21b_a3b_hellaswag.yaml

@@ -103,10 +103,6 @@ ci:
   recipe_owner: HuiyingLi
   time: "00:20:00"
   checkpoint_robustness:
-    check_source_load_parity: true
-    hf_kl_threshold: 5e-3
-    source_load_kl_threshold: 5e-3
-    source_load_cosine_threshold: 0.9995
     tokenizer_name: baidu/ERNIE-4.5-21B-A3B-PT
     hf_device_map_auto: true
     dataset.num_samples_limit: 500

examples/llm_finetune/gemma/gemma_3_270m_squad.yaml

@@ -96,7 +96,6 @@ ci:
   recipe_owner: HuiyingLi
   time: "00:20:00"
   checkpoint_robustness:
-    hf_kl_threshold: 1e-1
     tokenizer_name: google/gemma-3-270m
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500

examples/llm_finetune/gemma/gemma_3_270m_squad_peft.yaml

@@ -105,7 +105,6 @@ ci:
   known_issue_id: AM-154
   allow_failure: true
   checkpoint_robustness:
-    hf_kl_threshold: 1e-1
     tokenizer_name: google/gemma-3-270m
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500

examples/llm_finetune/glm/glm_4.7_flash_te_deepep.yaml

@@ -120,3 +120,12 @@ optimizer:
 
 ci:
   recipe_owner: hemildesai
+  nodes: 1
+  time: "00:30:00"
+  checkpoint_robustness:
+    # Keep the currently mismatched HF comparisons blocking until the model or
+    # reference path is fixed; an observed mismatch alone is not a reason to skip.
+    tokenizer_name: zai-org/GLM-4.7-Flash
+    hf_device_map_auto: true
+    dataset.num_samples_limit: 500
+    validation_dataset.num_samples_limit: 500

examples/llm_finetune/gpt_oss/customizer_gpt_oss_full_sft.yaml

@@ -87,7 +87,8 @@ parallelizer:
 ci:
   time: "00:30:00"
   checkpoint_robustness:
-    hf_kl_threshold: 5e-2
+    # Routed-MoE source parity measures 0.0064-0.0076 mean KL at 2K while
+    # p95 KL and cosine pass standard; retain every gate with the shared profile.
+    parity_tolerance_profile: relaxed
     tokenizer_name: openai/gpt-oss-20b
-    no_check_resume: true
     check_phantom_keys: true

examples/llm_finetune/gpt_oss/customizer_gpt_oss_full_sft_chat.yaml

@@ -511,6 +511,7 @@ parallelizer:
 ci:
   time: "00:30:00"
   checkpoint_robustness:
-    hf_kl_threshold: 5e-2
+    # Routed-MoE source parity measures 0.0064-0.0076 mean KL at 2K while
+    # p95 KL and cosine pass standard; retain every gate with the shared profile.
+    parity_tolerance_profile: relaxed
     tokenizer_name: openai/gpt-oss-20b
-    no_check_resume: true

examples/llm_finetune/gpt_oss/customizer_gpt_oss_peft.yaml

@@ -94,6 +94,7 @@ ci:
   time: "00:30:00"
   nproc_per_node: 1
   checkpoint_robustness:
-    hf_kl_threshold: 5e-2
+    # Routed-MoE source parity measures 0.0064-0.0076 mean KL at 2K while
+    # p95 KL and cosine pass standard; retain every gate with the shared profile.
+    parity_tolerance_profile: relaxed
     tokenizer_name: openai/gpt-oss-20b
-    no_check_resume: true

examples/llm_finetune/gpt_oss/customizer_gpt_oss_peft_packing.yaml

@@ -94,6 +94,7 @@ ci:
   time: "00:30:00"
   nproc_per_node: 1
   checkpoint_robustness:
-    hf_kl_threshold: 5e-2
+    # Routed-MoE source parity measures 0.0064-0.0076 mean KL at 2K while
+    # p95 KL and cosine pass standard; retain every gate with the shared profile.
+    parity_tolerance_profile: relaxed
     tokenizer_name: openai/gpt-oss-20b
-    no_check_resume: true

examples/llm_finetune/gpt_oss/gpt_oss_20b.yaml

@@ -118,14 +118,12 @@ optimizer:
 
 ci:
   recipe_owner: hemildesai
-  time: "00:15:00"
+  time: "00:30:00"
   node_multiplier: true
   vllm_deploy: true
   vllm_smoke_test: true
   checkpoint_robustness:
-    hf_kl_threshold: 1e-1
     tokenizer_name: openai/gpt-oss-20b
     check_phantom_keys: true
-    no_check_resume: true
     dataset.num_samples_limit: 500
     validation_dataset.num_samples_limit: 500

examples/llm_finetune/gpt_oss/gpt_oss_20b_peft.yaml

@@ -118,11 +118,12 @@ ci:
   vllm_deploy: true
   vllm_smoke_test: true
   recipe_owner: akoumpa
-  time: "00:15:00"
+  time: "00:20:00"
   checkpoint_robustness:
-    hf_kl_threshold: 5e-2
+    # PEFT source and reload comparisons consistently sit above the standard
+    # same-implementation envelope while remaining well inside the shared relaxed profile.
+    parity_tolerance_profile: relaxed
     tokenizer_name: openai/gpt-oss-20b
-    no_check_resume: true
     dataset.num_samples_limit: 500
     validation_dataset.num_samples_limit: 500
 

examples/llm_finetune/kimi/kimi_linear_48b_a3b_hellaswag.yaml

@@ -141,4 +141,10 @@ wandb:
 ci:
   recipe_owner: huiyingl
   nodes: 1
-  time: "01:00:00"
+  time: "00:40:00"
+  checkpoint_robustness:
+    tokenizer_name: moonshotai/Kimi-Linear-48B-A3B-Instruct
+    trust_remote_code: true
+    hf_device_map_auto: true
+    dataset.num_samples_limit: 500
+    validation_dataset.num_samples_limit: 500

examples/llm_finetune/llama3_1/customizer_llama_3_1_8b_full_sft_tp.yaml

@@ -80,5 +80,4 @@ loss_fn:
 ci:
   time: "00:30:00"
   checkpoint_robustness:
-    hf_kl_threshold: 1e-2
     tokenizer_name: meta-llama/Llama-3.1-8B-Instruct

examples/llm_finetune/llama3_2/customizer_llama_3_2_1b_full_sft.yaml

@@ -81,6 +81,5 @@ ci:
   time: "00:30:00"
   nproc_per_node: 1
   checkpoint_robustness:
-    hf_kl_threshold: 1e-2
     training_reproducibility_loss_threshold: 5e-3
     tokenizer_name: meta-llama/Llama-3.2-1B-Instruct

examples/llm_finetune/llama3_2/customizer_llama_3_2_1b_full_sft_chat.yaml

@@ -169,5 +169,4 @@ ci:
   time: "00:30:00"
   nproc_per_node: 1
   checkpoint_robustness:
-    hf_kl_threshold: 5e-3
     tokenizer_name: meta-llama/Llama-3.2-1B-Instruct

examples/llm_finetune/llama3_2/customizer_llama_3_2_1b_peft.yaml

@@ -88,5 +88,4 @@ ci:
   time: "00:30:00"
   nproc_per_node: 1
   checkpoint_robustness:
-    hf_kl_threshold: 5e-3
     tokenizer_name: meta-llama/Llama-3.2-1B-Instruct

examples/llm_finetune/llama3_2/customizer_llama_3_2_1b_peft_packing.yaml

@@ -88,5 +88,4 @@ ci:
   time: "00:30:00"
   nproc_per_node: 1
   checkpoint_robustness:
-    hf_kl_threshold: 5e-3
     tokenizer_name: meta-llama/Llama-3.2-1B-Instruct

examples/llm_finetune/llama3_2/llama3_2_1b_hellaswag.yaml

@@ -91,17 +91,11 @@ optimizer:
 
 ci:
   recipe_owner: akoumpa
-  time: "00:15:00"
+  time: "00:20:00"
   checkpoint_robustness:
-    check_source_load_parity: true
-    hf_kl_threshold: 5e-3
-    source_load_kl_threshold: 5e-3
-    source_load_cosine_threshold: 0.9995
     model.pretrained_model_name_or_path: meta-llama/Llama-3.2-3B-Instruct
     tokenizer_name: meta-llama/Llama-3.2-3B-Instruct
     cross_tp_size: 2
-    # Allow observed bf16 row-parallel reduction drift across 8-GPU TP reloads.
-    cross_tp_kl_threshold: 6e-3
     dataset.num_samples_limit: 500
     validation_dataset.num_samples_limit: 500
 

examples/llm_finetune/llama3_2/llama3_2_1b_hellaswag_peft.yaml

@@ -103,7 +103,6 @@ ci:
   recipe_owner: akoumpa
   time: "00:15:00"
   checkpoint_robustness:
-    hf_kl_threshold: 5e-3
     model.pretrained_model_name_or_path: meta-llama/Llama-3.2-3B-Instruct
     tokenizer_name: meta-llama/Llama-3.2-3B-Instruct
     check_fused_qkv_keys: true

examples/llm_finetune/llama3_2/llama_3_2_3b_instruct_squad.yaml

@@ -97,14 +97,13 @@ optimizer:
 ci:
   vllm_deploy: true
   recipe_owner: akoumpa
+  time: "00:30:00"
   known_issue_id: AM-154
   allow_failure: true
   checkpoint_robustness:
-    hf_kl_threshold: 5e-3
     model.pretrained_model_name_or_path: meta-llama/Llama-3.2-3B-Instruct
     tokenizer_name: meta-llama/Llama-3.2-3B-Instruct
     cross_tp_size: 2
-    cross_tp_kl_threshold: 5e-3
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500
 

examples/llm_finetune/llama3_2/llama_3_2_3b_instruct_squad_peft.yaml

@@ -103,12 +103,11 @@ optimizer:
 ci:
   recipe_owner: akoumpa
   vllm_deploy: true
+  time: "00:40:00"
   checkpoint_robustness:
-    hf_kl_threshold: 5e-3
     model.pretrained_model_name_or_path: meta-llama/Llama-3.2-3B-Instruct
     tokenizer_name: meta-llama/Llama-3.2-3B-Instruct
     cross_tp_size: 2
-    cross_tp_kl_threshold: 5e-3
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500
 

examples/llm_finetune/minimax_m2/minimax_m2.7_hellaswag_lora.yaml

@@ -113,23 +113,21 @@ optimizer:
 ci:
   # pp_size(1) * ep_size(32) = 32 GPUs => 4 nodes (8 H100/node).
   recipe_owner: hemildesai
-  time: "00:30:00"
+  # Allow the complete source, reload, and resume calibration matrix to finish.
+  time: "01:00:00"
   nodes: 4
   checkpoint_robustness:
     # The robustness default local batch is 2; with 32 ranks the global batch
     # must be at least 64 and divisible by 2 * 32.
     step_scheduler.global_batch_size: 64
-    # Keep each large-model load in a fresh process so the HF reload has all
-    # local GPU memory available after the distributed train/save phase.
-    process_isolation: true
-    # Exact adapter fingerprints and a successful HF+PEFT forward still run;
-    # live-LoRA cross-runtime logit parity is not a stable quantized reference.
-    skip_hf_logit_parity: true
+    # Process isolation is the harness default, leaving all local GPU memory
+    # available to the HF reload after the distributed train/save phase exits.
     # Use Transformers' built-in MiniMax M2 implementation for the HF+PEFT
     # reload; the checkpoint's remote code targets an older Transformers API.
     trust_remote_code: false
+    # Keep the cross-framework gates blocking: the current built-in-HF mismatch
+    # must be fixed or explicitly proven to be an invalid reference before it is skipped.
     tokenizer_name: MiniMaxAI/MiniMax-M2.7
     hf_device_map_auto: true
-    no_check_resume: true
     dataset.num_samples_limit: 500
     validation_dataset.num_samples_limit: 500

examples/llm_finetune/mistral/ministral3_3b_squad.yaml

@@ -108,23 +108,15 @@ ci:
   recipe_owner: akoumpa
   time: "00:15:00"
   checkpoint_robustness:
-    check_source_load_parity: true
-    hf_kl_threshold: 6e-3
-    # TP=2 BF16 reductions add bounded source-load drift; retain independent
-    # max/mean KL and cosine gates instead of skipping the FP8 parity check.
-    source_load_kl_threshold: 2.5e-2
-    source_load_mean_kl_threshold: 3e-3
-    source_load_cosine_threshold: 0.9995
     distributed.tp_size: 2
     tokenizer_name: mistralai/Ministral-3-3B-Instruct-2512
     cross_tp_size: 2
-    cross_tp_kl_threshold: 5e-3
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500
     # vision_tower params receive no grads from text-only batches, so Adam
     # never materialises per-param `step` state for them; DCP resume load is
     # strict and fails with `Missing key ... optim.state.model.vision_tower.*`.
-    no_check_resume: true
+    skip_resume: true
 
 # Uncomment and configure for W&B logging
 # wandb:

examples/llm_finetune/mistral/ministral3_3b_squad_peft.yaml

@@ -120,22 +120,14 @@ ci:
   recipe_owner: akoumpa
   time: "00:15:00"
   checkpoint_robustness:
-    check_source_load_parity: true
-    # LoRA merge stays within the TP=2-to-HF drift measured by source parity.
-    hf_kl_threshold: 2.5e-2
-    # TP=2 BF16 reductions add bounded source-load drift; retain independent
-    # max/mean KL and cosine gates instead of skipping the FP8 parity check.
-    source_load_kl_threshold: 2.5e-2
-    source_load_mean_kl_threshold: 3e-3
-    source_load_cosine_threshold: 0.9995
     distributed.tp_size: 2
     tokenizer_name: mistralai/Ministral-3-3B-Instruct-2512
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500
     # vision_tower params receive no grads from text-only batches, so Adam
     # never materialises per-param `step` state for them; DCP resume load is
     # strict and fails with `Missing key ... optim.state.model.vision_tower.*`.
-    no_check_resume: true
+    skip_resume: true
 
 # Uncomment and configure for W&B logging
 # wandb:

examples/llm_finetune/mistral/mistral_7b_hellaswag_fp8.yaml

@@ -120,16 +120,6 @@ ci:
   recipe_owner: HuiyingLi
   time: "00:25:00"
   checkpoint_robustness:
-    check_source_load_parity: true
-    hf_kl_threshold: 1e-2
-    # FP8 here is training-time quantization of a bf16 checkpoint (see the fp8
-    # section above), so Phase 0 bounds the FP8-prepared trainer model against
-    # the raw bf16 HF reference. This is an FP8-only exception; keep the
-    # non-quantized Phase 0 configs at the default strict mean KL threshold.
-    source_load_kl_threshold: 1e-2
-    source_load_mean_kl_threshold: 3e-3
-    source_load_cosine_threshold: 0.999
     tokenizer_name: mistralai/Mistral-7B-v0.1
-    no_check_resume: true
     dataset.num_samples_limit: 500
     validation_dataset.num_samples_limit: 500

examples/llm_finetune/mistral/mistral_7b_squad.yaml

@@ -104,11 +104,7 @@ ci:
   recipe_owner: HuiyingLi
   time: "00:20:00"
   checkpoint_robustness:
-    check_source_load_parity: true
-    hf_kl_threshold: 5e-3
-    source_load_kl_threshold: 5e-3
-    source_load_cosine_threshold: 0.9995
     tokenizer_name: mistralai/Mistral-7B-v0.1
-    no_check_resume: true
+    skip_resume: true
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500

examples/llm_finetune/mistral/mistral_nemo_2407_squad.yaml

@@ -101,12 +101,8 @@ ci:
   recipe_owner: HuiyingLi
   time: "00:25:00"
   checkpoint_robustness:
-    check_source_load_parity: true
-    hf_kl_threshold: 5e-3
-    source_load_kl_threshold: 5e-3
-    source_load_cosine_threshold: 0.9995
     tokenizer_name: mistralai/Mistral-Nemo-Base-2407
     hf_device_map_auto: true
-    no_check_resume: true
+    skip_resume: true
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500

examples/llm_finetune/mistral/mixtral-8x7b-v0-1_squad.yaml

@@ -107,14 +107,12 @@ ci:
   time: "00:40:00"
   nodes: 2
   checkpoint_robustness:
-    check_source_load_parity: true
-    hf_kl_threshold: 1e-2
-    source_load_kl_threshold: 1e-2
-    source_load_cosine_threshold: 0.9995
+    # PP buffers are fixed to the 1024-token packed-sequence size.
+    parity_sequence_length: 1024
     tokenizer_name: mistralai/Mixtral-8x7B-v0.1
     trust_remote_code: true
     hf_device_map_auto: true
-    no_check_resume: true
+    skip_resume: true
     dataset.limit_dataset_samples: 500
     validation_dataset.limit_dataset_samples: 500
   release: