{ "generation": "offline-derived; YAML cards remain authoritative", "models": [ { "architecture": "DeepseekV3ForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "entries": [ { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-hf-to-megatron-cpu", "expected_result": "A CPU import of the pinned blockwise-FP8 Hugging Face checkpoint must dequantize every mapped weight correctly, persist a reloadable Megatron checkpoint, and pass a complete tensor audit. This workflow is deferred by this card.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model deepseek-ai/DeepSeek-V3 --hf-revision e815299b0bcbac849fa540c768ef21845365c9eb --megatron-path work/model-verification/deepseek-v3/imported-megatron --torch-dtype bfloat16 --low-memory-save --distributed-timeout-minutes 240 --tp 1 --pp 4 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-hf-to-megatron-gpu", "expected_result": "The 32-GPU import completed all 4,583 mapping tasks, blockwise-dequantized the pinned FP8 source into BF16, and persisted a reloadable 32-shard distributed Megatron checkpoint. A strict paired export subsequently matched all 46,183 model tensors and 684,489,845,504 values exactly.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "b758d52cb3d12e3ca316d41d9527e6072a1c325b", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 4 --cpu-processes-per-node 8 --cpus-per-task 16 --mem 0 --exclusive --hf-model deepseek-ai/DeepSeek-V3 --hf-revision e815299b0bcbac849fa540c768ef21845365c9eb --megatron-path work/model-verification/deepseek-v3/imported-megatron/iter_0000000 --hf-path work/model-verification/deepseek-v3/cpu-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --tp 1 --pp 4 --ep 8 --etp 1 --distributed-timeout-minutes 240 --distributed-save --save-every-n-ranks 1 --no-progress" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-megatron-to-hf-cpu", "expected_result": "The 32-process distributed CPU export exits successfully and writes all 163 source-shaped shards. The index contains exactly 46,183 BF16 model tensors; an exhaustive audit matches all 684,489,845,504 values against the pinned source after blockwise dequantization of all 45,808 FP8 tensors and 59 other declared dtype normalizations. Transformers strictly reloads the output as DeepseekV3ForCausalLM with all 967 state tensors and no loading discrepancies.\n", "hardware": null, "last_verified": "2026-08-26", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model deepseek-ai/DeepSeek-V3 --hf-revision e815299b0bcbac849fa540c768ef21845365c9eb --megatron-path work/model-verification/deepseek-v3/imported-megatron/iter_0000000 --hf-path work/model-verification/deepseek-v3/hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --save-every-n-ranks 1 --distributed-timeout-minutes 240 --tp 1 --pp 4 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-megatron-to-hf-gpu", "expected_result": "The command exited successfully and wrote all 163 source-shaped shards. The index contained exactly 46,183 BF16 model tensors; an exhaustive 32-rank audit matched all 684,489,845,504 values against the pinned source after blockwise dequantization of all 45,808 FP8 tensors. The exported configuration instantiated natively as DeepseekV3ForCausalLM.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-manual-forward-pass", "expected_result": "A pinned first-token Hugging Face versus Megatron comparison must match the argmax token, reach cosine similarity of at least 0.99, and report maximum and mean absolute logit differences. This workflow is deferred by this card.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/inference/infer.sh --nodes 4 --gpus-per-node 8 --task legacy-full-prefix-generation --hf_model_path deepseek-ai/DeepSeek-V3 --hf-revision e815299b0bcbac849fa540c768ef21845365c9eb --megatron_model_path work/model-verification/deepseek-v3/imported-megatron/iter_0000000 --tp 1 --pp 4 --ep 8 --etp 1 --prompt \"Explain why the sky appears blue in one sentence.\" --max_new_tokens 32 --apply-chat-template --thinking-mode disabled --legacy-full-prefix" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-inference", "expected_result": "The distributed Megatron execution reloaded the verified BF16 checkpoint, produced finite logits, and generated exactly 32 tokens with deterministic greedy decoding, without an earlier EOS. The literal completion was \"The sky appears blue because shorter blue wavelengths of sunlight are scattered more effectively by the Earth's atmosphere than longer red wavelengths, a phenomenon known as Rayleigh scattering.\"\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-pretrain-gb300", "expected_result": "A bounded public-data DeepSeek V3 pretraining run must complete with finite loss, zero skipped or NaN iterations, all five metrics, a saved post-setup configuration, and a reloadable final checkpoint. Functional training is deferred by this card.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.GB300", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-sft-gb300", "expected_result": "A pinned-data full-SFT run must finish with finite loss, zero skipped or NaN iterations, all five metrics, a saved post-setup configuration, and a reloadable final checkpoint. Fine-tuning is deferred by this card.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.GB300", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-sft-export-inference-gb300", "expected_result": "A verified full-SFT checkpoint must export to Hugging Face, strictly reload, and produce a recorded deterministic checkpoint-backed completion. Training and post-SFT export verification are deferred by this card.\n", "hardware": "GB300", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB300", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-sft-long-context-gb300", "expected_result": "A dedicated packed long-context SFT run must complete with context parallelism, finite loss, zero skipped or NaN iterations, all five metrics, and a reloadable checkpoint. This workflow is deferred by this card.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.GB300", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-peft-gb300", "expected_result": "A DeepSeek V3 PEFT recipe with an audited model-native attention adapter target set must complete with finite loss, zero skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint. This workflow is deferred by this card.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.GB300", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "c3836699a2e006528a1338902874f96da0f5d219", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-checkpoint-resume-gb300", "expected_result": "A direct continuation must restore model, optimizer, scheduler, data-order, and RNG state, match declared sentinel losses, and save a reloadable checkpoint to a distinct output root. Resume verification is deferred by this card.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.GB300", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "79308d0d99fb8c09b3f1a22baa68a4ffa4f4ce1e", "command_topologies": [ { "gpus_per_node": "8", "nodes": "128", "recipe": "deepseek_v3_pretrain_1024gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 1024 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 128 --gpus-per-node 8 --recipe deepseek_v3_pretrain_1024gpu_h100_bf16_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/deepseek-v3/h100-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-pretrain-performance-h100", "expected_result": "On exactly 1,024 H100 GPUs, the canonical 50-step mock-data BF16 recipe completed at TP2/PP8/VP4/CP1/EP64/ETP1 and GBS/MBS 16384/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 61,756.130 ms and 276.030 model TFLOP/s/GPU, passing gates of at most 62,000 ms and at least 275 TFLOP/s/GPU. Mock data and forced expert balancing make this benchmark-only evidence, not convergence evidence.\n", "hardware": "H100", "last_verified": "2026-08-15", "metrics": { "final_loss": 6.815285, "initial_loss": 11.89496, "last_10_steps_model_tflops_per_gpu_avg": 276.03, "last_10_steps_step_time_ms_avg": 61756.13, "last_10_steps_tokens_per_second_per_gpu_avg": 1061.206 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.H100", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "79308d0d99fb8c09b3f1a22baa68a4ffa4f4ce1e", "command_topologies": [ { "gpus_per_node": "4", "nodes": "64", "recipe": "deepseek_v3_pretrain_256gpu_gb200_fp8mx_config", "sequence_length": "4096", "total_gpus": 256 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 64 --gpus-per-node 4 --recipe deepseek_v3_pretrain_256gpu_gb200_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/deepseek-v3/gb200-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-pretrain-performance-gb200", "expected_result": "On exactly 256 GB200 GPUs, the canonical 50-step mock-data MXFP8 recipe completed at TP1/PP4/VP4/CP1/EP64/ETP1 and GBS/MBS 4096/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 13,841.180 ms and 1,231.040 model TFLOP/s/GPU, passing gates of at most 14,000 ms and at least 1,200 TFLOP/s/GPU. Mock data, forced expert balancing, and reduced-precision optimizer moments make this benchmark-only evidence, not convergence evidence.\n", "hardware": "GB200", "last_verified": "2026-08-14", "metrics": { "final_loss": 6.311604, "initial_loss": 11.89246, "last_10_steps_model_tflops_per_gpu_avg": 1231.04, "last_10_steps_step_time_ms_avg": 13841.18, "last_10_steps_tokens_per_second_per_gpu_avg": 4734.856 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "79308d0d99fb8c09b3f1a22baa68a4ffa4f4ce1e", "command_topologies": [ { "gpus_per_node": "4", "nodes": "64", "recipe": "deepseek_v3_pretrain_256gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 256 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 64 --gpus-per-node 4 --recipe deepseek_v3_pretrain_256gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/deepseek-v3/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v3-pretrain-performance-gb300", "expected_result": "On exactly 256 GB300 GPUs, the canonical 50-step mock-data MXFP8 recipe completed at TP1/PP2/VP8/CP1/EP32/ETP1 and GBS/MBS 4096/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 10,667.300 ms and 1,597.310 model TFLOP/s/GPU, passing gates of at most 11,000 ms and at least 1,550 TFLOP/s/GPU. Mock data, forced expert balancing, and reduced-precision optimizer moments make this benchmark-only evidence, not convergence evidence.\n", "hardware": "GB300", "last_verified": "2026-08-14", "metrics": { "final_loss": 6.236621, "initial_loss": 11.89526, "last_10_steps_model_tflops_per_gpu_avg": 1597.31, "last_10_steps_step_time_ms_avg": 10667.3, "last_10_steps_tokens_per_second_per_gpu_avg": 6143.635 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "deepseek-ai/DeepSeek-V3", "hf_revision": "e815299b0bcbac849fa540c768ef21845365c9eb", "min_transformers_version": "5.8.0", "slug": "deepseek-v3", "source_card": "examples/model_verification_cards/deepseek-v3/card.yaml", "summary": "Distributed H100 GPU import, BF16 export, and deterministic inference are verified against the pinned blockwise-FP8 checkpoint. The canonical pretrain_performance.H100 1024-GPU BF16, pretrain_performance.GB200 256-GPU MXFP8, and pretrain_performance.GB300 256-GPU MXFP8 recipes are also verified. A distributed CPU export of the same pinned checkpoint is also verified. Timing and throughput metrics from future functional training items remain sanity checks rather than optimized performance results. CPU import, forward correlation, functional training, fine-tuning, and checkpoint-resume verification remain deferred.\n", "title": "deepseek_v3" }, { "architecture": "DeepseekV4ForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "07c5cc30d0510d437bfbbe7b12d9d98387295d89", "entries": [ { "bridge_commit": "07c5cc30d0510d437bfbbe7b12d9d98387295d89", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model deepseek-ai/DeepSeek-V4-Flash --hf-revision 60d8d70770c6776ff598c94bb586a859a38244f1 --megatron-path work/model-verification/dsv4-flash/import-cpu --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-hf-to-megatron-cpu", "expected_result": "CPU import requires approximately 570 GB of CPU RAM for the full BF16 weight materialisation (285B parameters x 2 bytes). Verification requires a high-memory node with enough additional headroom for conversion workspace and strict checkpoint reload.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "3ecb452af619bef88083e10a7738362df8f5d0bf", "command_topologies": [ { "expert_parallel": "4", "gpus_per_node": "4", "nodes": "1", "total_gpus": 4 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --ep 4 --hf-model deepseek-ai/DeepSeek-V4-Flash --hf-revision 60d8d70770c6776ff598c94bb586a859a38244f1 --megatron-path work/model-verification/dsv4-flash/import-gpu --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-hf-to-megatron-gpu", "expected_result": "The command exits successfully and creates a reloadable iter_0000000 Megatron checkpoint.\n", "hardware": null, "last_verified": "2026-08-04", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "07c5cc30d0510d437bfbbe7b12d9d98387295d89", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model deepseek-ai/DeepSeek-V4-Flash --hf-revision 60d8d70770c6776ff598c94bb586a859a38244f1 --megatron-path work/model-verification/dsv4-flash/import-cpu/iter_0000000 --hf-path work/model-verification/dsv4-flash/export-cpu --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-megatron-to-hf-cpu", "expected_result": "CPU export requires a high-memory node with enough headroom for the full BF16 weight set and tensor-merging workspace, followed by strict Hugging Face reload.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "unverified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "2859a3ee0f02743c54b390de91fee878c6de8909", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "2", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 2 --gpus-per-node 4 --tp 1 --pp 1 --ep 8 --etp 1 --hf-model deepseek-ai/DeepSeek-V4-Flash --hf-revision 60d8d70770c6776ff598c94bb586a859a38244f1 --megatron-path work/model-verification/dsv4-flash/import-gpu/iter_0000000 --hf-path work/model-verification/dsv4-flash/export-gpu --torch-dtype bfloat16 --export-weight-dtype bfloat16 --trust-remote-code --not-strict" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-megatron-to-hf-gpu", "expected_result": "Two GB200 nodes produced 46 complete safetensors shards containing all 35,020 expected non-scale source keys. BF16 export intentionally omitted 34,167 quantization-scale companions; 35,017 exported tensors are BF16 and the three I32 tid2eid routing tables exactly match their source values after the expected integer cast. Two independent Transformers processes each strictly reloaded all 1,500 model modules across four GPUs with no CPU or disk placement and produced the same bounded greedy token. --not-strict permits only the intentional scale omission; exact key, shard, dtype, routing-table value, and reload checks remain required correctness gates.\n", "hardware": null, "last_verified": "2026-08-21", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "07c5cc30d0510d437bfbbe7b12d9d98387295d89", "command_topologies": [ { "expert_parallel": "4", "gpus_per_node": "4", "nodes": "1", "pipeline_parallel": "1", "total_gpus": 4 } ], "commands": [ "./scripts/inference/infer.sh --task model-comparison --nodes 1 --gpus-per-node 4 --ep 4 --hf_model_path deepseek-ai/DeepSeek-V4-Flash --megatron_model_path work/model-verification/dsv4-flash/import-gpu/iter_0000000 --pp 1 --ep 4 --prompt \"The capital of France is the city of\" --hf-revision 60d8d70770c6776ff598c94bb586a859a38244f1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-manual-forward-pass", "expected_result": "The attempted four-GPU side-by-side load exceeded the available 768 GB of aggregate GPU memory. Verification requires a larger allocation sized from measured peak memory; no minimum working topology has been established.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "07c5cc30d0510d437bfbbe7b12d9d98387295d89", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "2", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/inference/infer.sh --nodes 2 --gpus-per-node 4 --task legacy-full-prefix-generation --legacy-full-prefix --hf_model_path deepseek-ai/DeepSeek-V4-Flash --hf-revision 60d8d70770c6776ff598c94bb586a859a38244f1 --megatron_model_path work/model-verification/dsv4-flash/import-gpu/iter_0000000 --prompt \"The capital of France is\" --max_new_tokens 32 --tp 1 --pp 1 --ep 8 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-inference", "expected_result": "Optimized KV-cache autoregressive inference conflicts with DSv4HybridAttention's requirement that inference_context be None. The maintained legacy full-prefix task disables that context and remains a plausible non-cached compatibility path, but the exact model command must complete deterministic greedy generation before this item is promoted.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "unverified", "variant": null, "workflow": "inference" }, { "bridge_commit": "dff6445009ebd054028a883ea6d84f9a16ad6b9c", "command_topologies": [ { "gpus_per_node": "4", "nodes": "16", "recipe": "deepseek_v4_flash_pretrain_64gpu_gb200_fp8mx_library_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_pretrain_64gpu_gb200_fp8mx_library_config --mode pretrain --dataset megatron-indexed --max_steps 100 --pretrained_checkpoint work/model-verification/dsv4-flash/import-gpu/iter_0000000 'dataset.blend=[[\"work/data/the-pile/my-gpt3_08_text_document\"],null]' dataset.path_to_cache=work/cache/the-pile dataset.num_workers=0 dataset.random_seed=1234 rng.seed=1234 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.finetune=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=true checkpoint.save_rng=true checkpoint.async_save=false --save_dir work/model-verification/dsv4-flash/pretrain-ref --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "deepseek-v4-flash-pretrain-gb200", "expected_result": "On 64 GB200 (16 nodes x 4 GPUs), TP1/PP4/VPP4/EP16/CP1 uses dense DP16, expert DP1, and GBS/MBS 256/1. The command loads the imported model weights while starting optimizer and RNG state fresh. It uses HybridEP natural routing, grouped GEMM, TE fused cross entropy, selective recompute over mhc and mla_up_proj, and MXFP8 parameter gather and gradient-buffer reuse without activation offload. Expert capacity, paged stash, CUDA graphs, and forced load balancing remain disabled. The run completed 100 finite LM/MTP-loss steps with no skipped or NaN iterations and wrote complete grouped-MXFP8 checkpoints containing model, optimizer, scheduler, data-order, and RNG state at steps 50 and 100. The item-specific Bridge revision pins the compatible Megatron-LM dev revision used by this workload without changing the card's default environment for other verification items.\n", "hardware": "GB200", "last_verified": "2026-08-27", "metrics": { "final_loss": 3.280722, "initial_loss": 7.250204, "last_10_steps_model_tflops_per_gpu_avg": 189.69, "last_10_steps_step_time_ms_avg": 7856.37, "last_10_steps_tokens_per_second_per_gpu_avg": 2085.441495 }, "precision": "fp8_mx", "source_pointer": "items.pretrain.GB200", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "07c5cc30d0510d437bfbbe7b12d9d98387295d89", "command_topologies": [ { "context_parallel": "1", "expert_parallel": "8", "gpus_per_node": "4", "nodes": "8", "pipeline_parallel": "4", "recipe": "deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config", "sequence_length": "1024", "total_gpus": 32 } ], "commands": [ "./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_gb200_config --mode sft --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "deepseek-v4-flash-sft-gb200", "expected_result": "On 32 GB200 (8 nodes x 4 GPUs), TP1/PP4/EP8/CP1 with dense DP8 and expert DP1, GBS/MBS 128/1. OpenMathInstruct-2 thinking data uses offline packing at seq_length=1024 with fixed token and cumulative-boundary shapes. The GB200 recipe uses selective recompute over moe, mhc, mla_up_proj, and layernorm, attention activation offload, HybridEP dispatch, and DSA kernel fusion without enabling the optional DSA indexer loss. The run completed 100 steps with finite LM and MTP losses, no skipped or NaN iterations, and a complete model checkpoint at step 100. A fresh process reloaded that checkpoint and completed finite step 101. moe_grouped_gemm=True is a recipe default. dist.distributed_timeout_minutes must be 180 or above to allow offline data packing (approximately 63 minutes) before training begins.\n", "hardware": "GB200", "last_verified": "2026-08-20", "metrics": { "final_loss": 0.2834835, "initial_loss": 0.9435847, "last_10_steps_model_tflops_per_gpu_avg": 42.18, "last_10_steps_step_time_ms_avg": 8575.85, "last_10_steps_tokens_per_second_per_gpu_avg": 477.620294, "peak_allocated_memory_gib": 184.96, "peak_reserved_memory_gib": 187.04 }, "precision": "bf16", "source_pointer": "items.sft.GB200", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "2859a3ee0f02743c54b390de91fee878c6de8909", "command_topologies": [ { "expert_parallel": "8", "gpus_per_node": "4", "nodes": "2", "total_gpus": 8 }, { "gpus_per_node": "4", "nodes": "2", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 2 --gpus-per-node 4 --ep 8 --hf-model deepseek-ai/DeepSeek-V4-Flash --hf-revision 60d8d70770c6776ff598c94bb586a859a38244f1 --megatron-path work/model-verification/dsv4-flash/sft-ref/iter_0000100 --hf-path work/model-verification/dsv4-flash/sft-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --trust-remote-code --not-strict", "./scripts/inference/infer.sh --task hf-inference --nodes 2 --gpus-per-node 4 --tasks-per-node 1 --hf-model work/model-verification/dsv4-flash/sft-export --prompt \"In one short sentence, explain why Paris is important to France.\" --max-new-tokens 32 --chat-template --disable-thinking --trust-remote-code --dtype bfloat16 --device-map auto --autocast --require-gpu-only" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-sft-export-inference-gb200", "expected_result": "The objective-preserving step-100 SFT checkpoint exported successfully on two GB200 nodes. Its 46 complete safetensors shards contain all 35,020 expected non-scale source keys: 35,017 BF16 tensors and three value-matched I32 tid2eid routing tables, with 34,167 quantization-scale companions intentionally omitted for BF16. Two maintained-verifier processes independently reloaded all 1,500 model modules across four GPUs each, reported no missing, unexpected, mismatched, or error loading entries, used no CPU or disk placement, and generated exactly 32 new tokens at the 32-token maximum with identical output. The literal completion was: \"Paris is important to France because it is the country's capital, a major economic and cultural center, and a symbol of French history, art, and architecture.\"\n", "hardware": "GB200", "last_verified": "2026-08-21", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB200", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "3ecb452af619bef88083e10a7738362df8f5d0bf", "command_topologies": [ { "context_parallel": "2", "expert_parallel": "8", "gpus_per_node": "4", "nodes": "16", "pipeline_parallel": "4", "recipe": "deepseek_v4_flash_sft_openmath_thinking_packed_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_sft_openmath_thinking_packed_config --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/sft-long-context-ref --save_interval 100 --max_steps 100 model.pipeline_model_parallel_size=4 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' model.context_parallel_size=2 model.cp_partition_mode=contiguous model.expert_model_parallel_size=8 model.moe_grouped_gemm=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 dataset.seq_length=1024 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "offline" }, "entry_id": "deepseek-v4-flash-sft-long-context-gb200", "expected_result": "CP=2 with contiguous partitioning and offline-packed OpenMathInstruct-2 at seq_length=1024 demonstrates sequence packing and context parallelism working together over 100 training steps with finite loss and no skipped or NaN iterations.\n", "hardware": "GB200", "last_verified": "2026-08-07", "metrics": { "final_loss": 0.2837, "initial_loss": 0.94383, "last_10_steps_model_tflops_per_gpu_avg": 13.62, "last_10_steps_step_time_ms_avg": 13806.5, "last_10_steps_tokens_per_second_per_gpu_avg": 148.336 }, "precision": "bf16", "source_pointer": "items.sft_long_context.GB200", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "043d9cdaa1ac0cb52437d8d417a9f3150dfcae3b", "command_topologies": [ { "context_parallel": "1", "expert_parallel": "8", "gpus_per_node": "4", "nodes": "8", "pipeline_parallel": "4", "recipe": "deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config", "sequence_length": "1024", "total_gpus": 32 } ], "commands": [ "./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe deepseek_v4_flash_peft_openmath_thinking_packed_gb200_config --mode lora --step-func dsv4_step --pretrained_checkpoint work/models/deepseek-v4-flash-megatron --save_dir work/model-verification/dsv4-flash/peft-ref --save_interval 100 --max_steps 100 --seq_length 1024 --pipeline_model_parallel_size 4 --context_parallel_size 1 --expert_model_parallel_size 8 'model.pipeline_model_parallel_layout=Et*11|t*11|t*11|t*10mL' scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 rng.seed=5678 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.save_optim=false ddp.overlap_grad_reduce=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=180" ], "dimensions": {}, "enabled_features": { "cuda_graph": { "implementation": "transformer_engine", "scopes": [ "moe_router", "moe_preprocess" ] }, "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "deepseek-v4-flash-peft-gb200", "expected_result": "On 32 GB200 (8 nodes x 4 GPUs), TP1/PP4/EP8/CP1 with dense DP8 and expert DP1, GBS/MBS 128/1. The recipe applies rank-32, alpha-32 LoRA with zero dropout to linear_q_down_proj, linear_q_up_proj, linear_kv_proj, linear_proj, linear_fc1, and linear_fc2. Routed experts use separate per-expert adapters; shared experts are also adapted. The run preserves the packed OpenMath thinking data, objective, natural routing, seed, and warmup/decay horizon used by the verified SFT item while freezing the base model and using PEFT's own 1e-4 peak learning rate. It disables activation recompute and offload because the reduced PEFT training-state footprint fits with 53.712 GiB peak allocated memory. Transformer Engine CUDA graphs cover moe_router and moe_preprocess, while HybridEP uses 32 flex-dispatcher SMs, 8 ranks per NVLink domain, 128-token combine chunks, a 72-GPU domain, and MNNVL. All 100 steps completed with finite LM/MTP losses and zero skipped or NaN iterations. LM/MTP losses were 0.9435847/0.2250465 at step 1 and 0.2914867/0.02638751 at step 100. The adapter-only checkpoint reloaded after the same base checkpoint in a fresh process and completed finite step 101 with LM loss 0.2827368 and MTP loss 0.02594211.\n", "hardware": "GB200", "last_verified": "2026-08-25", "metrics": { "final_loss": 0.2914867, "initial_loss": 0.9435847, "last_10_steps_model_tflops_per_gpu_avg": 52.0, "last_10_steps_step_time_ms_avg": 6944.33, "last_10_steps_tokens_per_second_per_gpu_avg": 589.83372, "peak_allocated_memory_gib": 53.712, "peak_reserved_memory_gib": 53.953 }, "precision": "bf16", "source_pointer": "items.peft.GB200", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "dff6445009ebd054028a883ea6d84f9a16ad6b9c", "command_topologies": [ { "gpus_per_node": "4", "nodes": "16", "recipe": "deepseek_v4_flash_pretrain_64gpu_gb200_fp8mx_library_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe deepseek_v4_flash_pretrain_64gpu_gb200_fp8mx_library_config --mode pretrain --dataset megatron-indexed --max_steps 100 'dataset.blend=[[\"work/data/the-pile/my-gpt3_08_text_document\"],null]' dataset.path_to_cache=work/cache/the-pile dataset.num_workers=0 dataset.random_seed=1234 rng.seed=1234 scheduler.lr_warmup_iters=10 scheduler.lr_decay_iters=100 validation.eval_interval=0 validation.eval_iters=0 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.exit_on_missing_checkpoint=false checkpoint.finetune=false checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.save_optim=true checkpoint.save_rng=true checkpoint.async_save=false --load_dir work/model-verification/dsv4-flash/pretrain-ref --save_dir work/model-verification/dsv4-flash/pretrain-resumed --save_interval 50 checkpoint.ckpt_step=50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-checkpoint-resume-gb200", "expected_result": "The direct 64-GPU TP1/PP4/VPP4/EP16 continuation restores model, optimizer, scheduler, data-order, and RNG state from the complete step-50 reference checkpoint, starts at step 51, and finishes at step 100 in a distinct output root. All 50 resumed steps are finite with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and a complete full-state step-100 checkpoint is saved. Step-51 resumed/reference LM loss is 4.028752/4.028773, an absolute difference of 0.000021. Step-100 resumed/reference LM loss is 3.261145/3.280722, an absolute difference of 0.019577. Both sentinels pass the declared 1% relative and 1e-6 absolute gate.\n", "hardware": "GB200", "last_verified": "2026-08-27", "metrics": { "final_loss": 3.261145, "initial_loss": 4.028752, "last_10_steps_model_tflops_per_gpu_avg": 186.64, "last_10_steps_step_time_ms_avg": 7995.85, "last_10_steps_tokens_per_second_per_gpu_avg": 2049.062951 }, "precision": "fp8_mx", "source_pointer": "items.checkpoint_resume.GB200", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "3ecb452af619bef88083e10a7738362df8f5d0bf", "command_topologies": [ { "gpus_per_node": "4", "nodes": "32", "recipe": "deepseek_v4_flash_pretrain_128gpu_gb200_fp8mx_config", "total_gpus": 128 } ], "commands": [ "./scripts/training/train.sh --nodes 32 --gpus-per-node 4 --recipe deepseek_v4_flash_pretrain_128gpu_gb200_fp8mx_config --max_steps 50 scheduler.lr_warmup_iters=5 scheduler.lr_decay_iters=50 validation.eval_interval=0 validation.eval_iters=0 checkpoint.exit_on_missing_checkpoint=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-pretrain-performance-gb200", "expected_result": "On 128 GB200, PP1/EP64/CP1/TP1/DP2 with GBS/MBS 2048/1, FP8-MX, HybridEP dispatch, and full-iteration CUDA graphs completes 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. This benchmark uses forced routing, a static expert-rank capacity, paged stash, and a different batch/objective contract, so its metrics must not be compared with natural-routing library pretraining as convergence evidence.\n", "hardware": "GB200", "last_verified": "2026-08-06", "metrics": { "final_loss": 3.62683, "initial_loss": 13.5875, "last_10_steps_model_tflops_per_gpu_avg": 731.4, "last_10_steps_step_time_ms_avg": 8134.6, "last_10_steps_tokens_per_second_per_gpu_avg": 8056.45 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "c93251151adeeadbae3ff2a2bf5ee7a1c34cff01", "command_topologies": [ { "gpus_per_node": "4", "nodes": "32", "recipe": "deepseek_v4_flash_pretrain_128gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 128 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 32 --gpus-per-node 4 --recipe deepseek_v4_flash_pretrain_128gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/deepseek-v4-flash/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "deepseek-v4-flash-pretrain-performance-gb300", "expected_result": "On exactly 128 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 2048/1, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.59616 to 3.160061; the final ten steps average 7840.320 ms, 759.280 TFLOP/s/GPU, and 8358.842 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB300", "last_verified": "2026-08-13", "metrics": { "final_loss": 3.160061, "initial_loss": 12.59616, "last_10_steps_model_tflops_per_gpu_avg": 759.28, "last_10_steps_step_time_ms_avg": 7840.32, "last_10_steps_tokens_per_second_per_gpu_avg": 8358.842 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "deepseek-ai/DeepSeek-V4-Flash", "hf_revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "min_transformers_version": "5.12.0", "slug": "deepseek-v4-flash", "source_card": "examples/model_verification_cards/deepseek-v4-flash/card.yaml", "summary": "Performance scope: pretrain_performance.GB200 and pretrain_performance.GB300 use their tuned canonical 128-GPU MXFP8 recipes. Timing and throughput metrics from functional training items are sanity checks rather than optimized performance results.\nThe Megatron-backed DeepSeek V4 workflows in this card require the Megatron-Core dev branch; the default MCore submodule revision in Megatron Bridge r0.6.0 is insufficient. Follow the public setup instructions in examples/models/deepseek_v4/README.md.\nDeepSeek-V4-Flash verification covers GPU import at the item-pinned Bridge revision. The verified 64-GPU GB200 MXFP8 library pretraining run uses PP4/VPP4/EP16 with imported model weights, natural routing, grouped GEMM, TE fused cross entropy, and selective recompute without activation offload. It completed 100 finite steps with full-state checkpoints at steps 50 and 100. A direct step-50 continuation restored optimizer, scheduler, data-order, and RNG state, completed steps 51-100, and passed both loss-sentinel comparisons. A capacity-1.5 performance screen averaged 206.56 TFLOP/s/GPU over steps 2-8, but dropped 66.16% of expert assignments and 64.23% of routing probability mass; that result is benchmark-only and is excluded from the natural-routing library recipe and verification metrics. CPU conversion remains unverified because the available nodes lacked sufficient RAM. Standard Megatron KV-cache inference is unsupported for DSV4 hybrid attention; inference uses the HF-native path after export.\nThe GB200 packed SFT recipe uses grouped GEMM, static packed-sequence shapes, selective whole-MoE recompute, and attention activation offload while preserving the base SFT objective. Its final objective-preserving configuration completed 100 finite steps and a fresh-process checkpoint reload. GPU export of both the imported model and the step-100 SFT checkpoint produced exact BF16 inventories and passed independent strict GPU-only HF reloads. The public post-SFT workflow also produced identical bounded greedy completions from two independent reloads. A separate 100-step CP2 run verifies offline packing with contiguous context parallelism.\nThe GB200 packed PEFT recipe applies rank-32 LoRA to the DeepSeek MLA projections and to separate routed/shared expert FC1/FC2 projections. Its execution path uses scoped Transformer Engine CUDA graphs and tuned HybridEP dispatch. It completed 100 finite steps, saved an adapter-only checkpoint, and reloaded that checkpoint in a fresh process for finite step 101.\n", "title": "deepseek_v4_flash" }, { "architecture": "Gemma3ForConditionalGeneration", "base_container": "nvcr.io/nvidia/pytorch:26.04-py3", "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "entries": [ { "bridge_commit": "2d0175b116af6cb785bb8efab7ee053435496266", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model google/gemma-3-4b-it --hf-revision 093f9f388b31de276ce2de164bdc2081324b9767 --megatron-path work/model-verification/gemma-3-4b-it/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-hf-to-megatron-cpu", "expected_result": "Exact-revision BF16 CPU import loaded all 781 Bridge tensors and saved one reloadable DCP shard. An independent Gloo reload mapped all 883 source tensors with exact keys, shapes, dtypes, and torch.equal values, including all 437 legacy vision-tower keys.\n", "hardware": null, "last_verified": "2026-07-24", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "2d185b9619c302281eb1a3219b3c7afaee940afc", "command_topologies": [ { "expert_parallel": "1", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "2", "total_gpus": 4 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --hf-model google/gemma-3-4b-it --hf-revision 093f9f388b31de276ce2de164bdc2081324b9767 --megatron-path work/model-verification/gemma-3-4b-it/imported-megatron --torch-dtype bfloat16 --tp 2 --pp 2 --ep 1 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-hf-to-megatron-gpu", "expected_result": "Exact-revision distributed BF16 import completed with TP2/PP2/EP1/ETP1. All 781 Bridge tensors loaded and all four ranks saved a reloadable checkpoint with the vision tower and multimodal projector on the owning pipeline stage. Independent TP2/PP2 and Gloo reloads retained FusedAttention and matched all 883 mapped tensors exactly.\n", "hardware": null, "last_verified": "2026-07-30", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model google/gemma-3-4b-it --hf-revision 093f9f388b31de276ce2de164bdc2081324b9767 --megatron-path work/model-verification/gemma-3-4b-it/cpu-megatron/iter_0000000 --hf-path work/model-verification/gemma-3-4b-it/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-megatron-to-hf-cpu", "expected_result": "The exact-revision CPU export exits successfully and writes all 883 tensors, 4,300,079,472 values, and 8,600,158,944 tensor-payload bytes. Keys, shapes, dtypes, and values are bitwise identical to the pinned BF16 source with no dtype conversions, including all 437 legacy vision_tower.vision_model.* tensors. Transformers strictly reloads the output natively as Gemma3ForConditionalGeneration with all 884 state entries and no loading discrepancies.\n", "hardware": null, "last_verified": "2026-08-26", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "2d185b9619c302281eb1a3219b3c7afaee940afc", "command_topologies": [ { "expert_parallel": "1", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "2", "total_gpus": 4 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --hf-model google/gemma-3-4b-it --hf-revision 093f9f388b31de276ce2de164bdc2081324b9767 --megatron-path work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --hf-path work/model-verification/gemma-3-4b-it/hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 2 --pp 2 --ep 1 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-megatron-to-hf-gpu", "expected_result": "TP2/PP2 distributed export completed without a missing PP owner. Source and export contained the same 883 keys with exact shapes, dtypes, and torch.equal values, including all 437 legacy vision_tower.vision_model.* tensors. Transformers exposes 884 logical state entries after reload because tied lm_head.weight aliases the single stored language_model.model.embed_tokens.weight tensor; no extra tensor is serialized. Native Gemma3ForConditionalGeneration strict reload reported no missing, unexpected, mismatched, or errored tensors.\n", "hardware": null, "last_verified": "2026-07-30", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "command_topologies": [ { "expert_parallel": "1", "expert_tensor_parallel": "1", "pipeline_parallel": "1", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=1 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path google/gemma-3-4b-it --hf-revision 093f9f388b31de276ce2de164bdc2081324b9767 --megatron_model_path work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --model_class Gemma3ForConditionalGeneration --image_path work/model-verification/gemma-3-4b-it/candy.JPG --prompt \"What animal is on the candy?\" --tp 1 --pp 1 --ep 1 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-manual-forward-pass", "expected_result": "The exact-revision TP1 image-and-text comparison selected Transformer Engine FusedAttention. Token match: true; Hugging Face and Megatron both produced next token ID 22515 (\"Based\"). Cosine similarity: 0.990387, passing the required 0.99 threshold; maximum and mean absolute logit differences were 6.234131 and 1.007509 and are report-only diagnostics.\n", "hardware": null, "last_verified": "2026-07-30", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "2d185b9619c302281eb1a3219b3c7afaee940afc", "command_topologies": [ { "expert_parallel": "1", "expert_tensor_parallel": "1", "pipeline_parallel": "2", "tensor_parallel": "2" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py --hf_model_path google/gemma-3-4b-it --hf-revision 093f9f388b31de276ce2de164bdc2081324b9767 --megatron_model_path work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --image_path work/model-verification/gemma-3-4b-it/candy.JPG --prompt \"What animal is on the candy?\" --max_new_tokens 32 --tp 2 --pp 2 --ep 1 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-inference", "expected_result": "One exact-revision TP2/PP2 BF16 greedy multimodal run selected\nFusedAttention and generated exactly 32 tokens without reaching EOS. The\npublic input was p-blog/candy.JPG at documentation-images revision\n44c3249461c5a5336585d1e31e0a4ce7fce06a5a with SHA-256\nfc417c899e94f8df465b7541c5a70f0eebb85c414d06345f0b290c061eccc84c.\nThe JSON-escaped literal completion was: \"Based on the image, the animal\non the candy is a **turtle**. \\n\\nYou can see a clear depiction of a\nturtle shell on each of the candies\"", "hardware": null, "last_verified": "2026-07-30", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-pretrain-all", "expected_result": "Megatron Bridge does not provide a public Gemma 3 VL 4B pretraining recipe. The available exact-model recipes cover SFT and PEFT, so a supported pretraining command and checkpoint contract cannot be stated.\n", "hardware": "all", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.pretrain.all", "status": "unsupported", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "recipe": "gemma3_vl_4b_sft_config", "sequence_length": "4096", "total_gpus": 1 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gemma3_vl_4b_sft_config --mode sft --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:\"7f0115a4b758a71d6473b8d085751692da2fef98\"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-3-4b-it/sft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-sft-h100", "expected_result": "Pending exact-revision CORD-v2 full-SFT verification with the public Gemma 3 VL 4B H100 recipe. The run must complete at least ten optimizer steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final full-model checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-sft-export-inference-h100", "expected_result": "Pending the full-SFT checkpoint. Export must reload natively with exact keys, shapes, and dtypes, then one deterministic greedy multimodal Hugging Face generation must record its exact generated-token count and literal completion.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-sft-long-context-h100", "expected_result": "Gemma 3 VL image-bidirectional attention uses dense local/global additive biases with FusedAttention. The provider rejects context parallelism greater than one, and the public recipes disable in-batch THD packing, so there is no supported exact-model long-context SFT contract.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.sft_long_context.H100", "status": "unsupported", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "recipe": "gemma3_vl_4b_peft_config", "sequence_length": "4096", "total_gpus": 1 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gemma3_vl_4b_peft_config --mode lora --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:\"7f0115a4b758a71d6473b8d085751692da2fef98\"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-3-4b-it/peft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-peft-h100", "expected_result": "Pending exact-revision CORD-v2 LoRA verification with the public Gemma 3 VL 4B H100 recipe. The run must complete at least ten optimizer steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "330f87c0a391eb2770cbf2306352cc807211d7ba", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-3-4b-it-checkpoint-resume-all", "expected_result": "Checkpoint-resume verification depends on a supported pretraining baseline. Because Megatron Bridge has no public Gemma 3 VL 4B pretraining recipe, there is no supported reference run or resume checkpoint contract for this model.\n", "hardware": "all", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.checkpoint_resume.all", "status": "unsupported", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "google/gemma-3-4b-it", "hf_revision": "093f9f388b31de276ce2de164bdc2081324b9767", "min_transformers_version": "5.8.0", "slug": "gemma-3-4b-it", "source_card": "examples/model_verification_cards/gemma-3-4b-it/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Gemma 3 4B Instruct exact-checkpoint CPU and TP2/PP2 GPU import, CPU and TP2/PP2 GPU export, manual multimodal forward parity, and deterministic TP2/PP2 inference are verified. SFT, post-SFT export/inference, and PEFT remain pending. Pretraining and checkpoint resume are unsupported because there is no public exact-model pretraining recipe. Long-context SFT is unsupported because the required dense local/global attention biases currently exclude context parallelism and in-batch THD packing. Existing toy-model tests are not used as exact-model evidence.\n", "title": "gemma_3_4b_it" }, { "architecture": "Gemma4ForConditionalGeneration", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "125695a5f134203d2aef7a5dd1465cfa3608b4f2", "entries": [ { "bridge_commit": "a0e186550b3dc5f9b36f98c9bd050994f0b3fdc6", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model google/gemma-4-26B-A4B-it --hf-revision 4d7ae4984b7db7de8f8457170b3f1a419ee76d52 --megatron-path work/model-verification/gemma-4-26b-a4b-it/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-hf-to-megatron-cpu", "expected_result": "The command completed all 8,548 conversion mappings and saved iter_0000000. All 383 tensor entries in the distributed-checkpoint metadata were torch.bfloat16. The run configuration recorded bf16=true, fp16=false, and BF16 parameter and autocast dtypes. The artifact reloaded successfully for CPU export.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "a0e186550b3dc5f9b36f98c9bd050994f0b3fdc6", "command_topologies": [ { "expert_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "4", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model google/gemma-4-26B-A4B-it --hf-revision 4d7ae4984b7db7de8f8457170b3f1a419ee76d52 --megatron-path work/model-verification/gemma-4-26b-a4b-it/imported-megatron --torch-dtype bfloat16 --tp 4 --pp 2 --ep 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-hf-to-megatron-gpu", "expected_result": "The command completed successfully at TP4/PP2/EP1. It loaded 8,548 conversion mappings from the immutable Hugging Face revision and saved the full multimodal checkpoint as iter_0000000. All 383 tensor entries in the distributed-checkpoint metadata were torch.bfloat16. The artifact then reloaded successfully for distributed export and direct-model inference.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "8e9fa9e2aaf4d89538b0c0ff15bb837d14b2af47", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model google/gemma-4-26B-A4B-it --hf-revision 4d7ae4984b7db7de8f8457170b3f1a419ee76d52 --megatron-path work/model-verification/gemma-4-26b-a4b-it/cpu-megatron/iter_0000000 --hf-path work/model-verification/gemma-4-26b-a4b-it/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-megatron-to-hf-cpu", "expected_result": "The command completed all 8,548 conversion mappings and wrote 11 BF16 shards. All 1,013 exported tensor entries matched the immutable source exactly in key, shape, dtype, and value. Native Gemma4ForConditionalGeneration reload reported model.dtype=torch.bfloat16 with no missing, unexpected, mismatched, or error keys.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "a0e186550b3dc5f9b36f98c9bd050994f0b3fdc6", "command_topologies": [ { "expert_parallel": "4", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model google/gemma-4-26B-A4B-it --hf-revision 4d7ae4984b7db7de8f8457170b3f1a419ee76d52 --megatron-path work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --hf-path work/model-verification/gemma-4-26b-a4b-it/hf-export-verified --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 2 --pp 1 --ep 4 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-megatron-to-hf-gpu", "expected_result": "Distributed TP2/PP1/EP4 export completed all 2,788 conversion mappings. The exported config preserved sliding-attention rope_theta 10000 and full-attention rope_theta 1000000. All 1,013 exported tensor entries were BF16 and matched the immutable source exactly in key, shape, dtype, and value. Native Gemma4ForConditionalGeneration reload reported model.dtype=torch.bfloat16 with no missing, unexpected, mismatched, or error keys.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "125695a5f134203d2aef7a5dd1465cfa3608b4f2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-manual-forward-pass", "expected_result": "No valid prompt next-token comparison is claimed. The 2026-07-23 TP2 run processed 285 real input tokens, padded them to 286, and compared the final padding position on both Hugging Face and Megatron. Its matching token and 0.999844 cosine similarity therefore do not satisfy the manual forward gate. Verification requires either an unpadded final position or a helper that selects the final real token before reporting next-token agreement and cosine similarity of at least 0.99.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "125695a5f134203d2aef7a5dd1465cfa3608b4f2", "command_topologies": [ { "expert_parallel": "4", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 8 --task vlm-generation --hf_model_path google/gemma-4-26B-A4B-it --hf-revision 4d7ae4984b7db7de8f8457170b3f1a419ee76d52 --megatron_model_path work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --image_path work/model-verification/gemma-4-26b-a4b-it/candy.JPG --prompt \"Describe the candy image in detail, including the animal, colors, text, and packaging.\" --max_new_tokens 32 --tp 2 --pp 1 --ep 4 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-inference", "expected_result": "On 2026-08-24 one synchronous 8-H100 BF16 greedy run loaded the verified checkpoint at TP2/PP1/EP4/ETP1 and completed exactly 32 generation steps under the 32-token maximum without EOS. The literal completion was \"A close-up, high-angle shot shows a person's left hand holding four small, round, colorful candies. The hand is slightly open, with\". The input was p-blog/candy.JPG from huggingface/documentation-images at revision 44c3249461c5a5336585d1e31e0a4ce7fce06a5a, with SHA-256 fc417c899e94f8df465b7541c5a70f0eebb85c414d06345f0b290c061eccc84c.\n", "hardware": null, "last_verified": "2026-08-24", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "125695a5f134203d2aef7a5dd1465cfa3608b4f2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-pretrain-all", "expected_result": "Megatron Bridge does not publish a pretraining recipe for this Gemma 4 conditional-generation variant; the current model package provides SFT and PEFT recipes only.\n", "hardware": "all", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.pretrain.all", "status": "unsupported", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "f3130a0e37c78945ecf98b301dfc90505c3c305f", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "gemma4_vl_26b_sft_config", "sequence_length": "4096", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gemma4_vl_26b_sft_config --mode sft --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:\"7f0115a4b758a71d6473b8d085751692da2fef98\"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-4-26b-a4b-it/sft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/gemma-4-26b-a4b-it/sft-post-setup.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-sft-h100", "expected_result": "On 2026-08-11 the single-node TP4/PP1/EP8/ETP1 run loaded the immutable BF16 checkpoint and real CORD-v2 revision, then completed exactly 10 optimizer steps at MBS1 and GBS32. The persisted post-setup config confirmed a frozen vision encoder, trainable projection and language model, expandable CUDA allocator segments, and full-layer uniform recompute with one layer per recompute unit directly from the recipe defaults, while CUDA graphs and CPU offload remained disabled. Loss moved from 1.282612 to 0.09127883, final grad norm was 3.090, and all 10 steps reported zero skipped and zero NaN iterations. After iteration 2, logger-reported ranks returned to 62.134-62.145 GiB allocated, while cumulative peak allocated and reserved memory remained at or below 65.223 and 72.356 GiB. The run saved a complete iter_0000010 torch_dist checkpoint with eight rank shards, finalized metadata, run config, training state, and a latest-checkpoint tracker containing iteration 10.\n", "hardware": "H100", "last_verified": "2026-08-11", "metrics": { "final_loss": 0.09127883, "initial_loss": 1.282612, "last_10_steps_model_tflops_per_gpu_avg": 32.2, "last_10_steps_step_time_ms_avg": 15936.25, "last_10_steps_tokens_per_second_per_gpu_avg": 1028.096, "peak_allocated_memory_gib": 65.223, "peak_reserved_memory_gib": 72.356 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "125695a5f134203d2aef7a5dd1465cfa3608b4f2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-sft-export-inference-h100", "expected_result": "SFT produced a verified iter_0000010 checkpoint, but export, strict Hugging Face reload, and post-SFT inference have not been run. This item requires the checkpoint to export, strictly reload as Gemma4ForConditionalGeneration, and complete one deterministic greedy VLM generation with a recorded literal completion.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "125695a5f134203d2aef7a5dd1465cfa3608b4f2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-sft-long-context-h100", "expected_result": "No Gemma 4 VL long-context recipe or run was verified. A future workload must establish sequence-packing and context-parallel settings, complete at least 10 optimizer steps with finite loss and no skipped or NaN iterations, report all five required metrics, and save a reloadable final checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "df1b978e751753707100417cb4e89645a23537f8", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "recipe": "gemma4_vl_26b_peft_config", "sequence_length": "4096", "total_gpus": 4 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe gemma4_vl_26b_peft_config --mode lora --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:\"7f0115a4b758a71d6473b8d085751692da2fef98\"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-4-26b-a4b-it/peft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/gemma-4-26b-a4b-it/peft-post-setup.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-peft-h100", "expected_result": "On 2026-08-04 the 4-GPU TP2/PP1/EP4/ETP1 LoRA run loaded the immutable BF16 checkpoint and real CORD-v2 revision, then completed exactly 10 optimizer steps at MBS1 and GBS32. The persisted post-setup config recorded rank-32 adapters with alpha 32 and zero dropout on linear_qkv, linear_proj, linear_fc1, and linear_fc2. Runtime transformation froze the base model and reported 22,272,000 trainable adapter parameters, 0.30% of the local model shard. Loss moved from 1.288328 to 0.1072377, final grad norm was 22.994, and every step reported zero skipped and zero NaN iterations. Across all four ranks, allocated memory returned to 14.582 GiB after each step, while cumulative peak allocated and reserved memory remained at or below 44.252 and 45.637 GiB. The run saved a complete iter_0000010 adapter checkpoint with four rank shards, finalized metadata, training state, and a latest-checkpoint tracker containing iteration 10.\n", "hardware": "H100", "last_verified": "2026-08-04", "metrics": { "final_loss": 0.1072377, "initial_loss": 1.288328, "last_10_steps_model_tflops_per_gpu_avg": 65.01, "last_10_steps_step_time_ms_avg": 14163.25, "last_10_steps_tokens_per_second_per_gpu_avg": 2313.593, "peak_allocated_memory_gib": 44.252, "peak_reserved_memory_gib": 45.637 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "125695a5f134203d2aef7a5dd1465cfa3608b4f2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "gemma-4-26b-a4b-it-checkpoint-resume-all", "expected_result": "Megatron Bridge does not provide a Gemma 4 VLM pretraining recipe, so there is no supported reference run, middle checkpoint, optimizer/RNG state contract, or model-wide checkpoint-resume workflow to verify.\n", "hardware": "all", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.checkpoint_resume.all", "status": "unsupported", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "google/gemma-4-26B-A4B-it", "hf_revision": "4d7ae4984b7db7de8f8457170b3f1a419ee76d52", "min_transformers_version": "5.8.0", "slug": "gemma-4-26b-a4b-it", "source_card": "examples/model_verification_cards/gemma-4-26b-a4b-it/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Gemma 4 26B-A4B-it BF16 CPU and GPU import and exact CPU and GPU export are verified. Deterministic multimodal inference is verified on one 8-GPU H100 node with TP2/PP1/EP4/ETP1. Manual forward comparison remains unverified because the recorded run compared a tensor-parallel padding position rather than the prompt's next token. Decoder-focused SFT is verified on one 8-GPU H100 node with TP4/PP1/EP8/ETP1, a frozen vision encoder, and full-layer uniform activation recompute with one layer per recompute unit. The real CORD-v2 run completed 10 optimizer steps with stable memory and saved iter_0000010. LoRA PEFT is also verified on four H100 GPUs with TP2/PP1/EP4/ETP1 and a complete adapter checkpoint. Long-context SFT is pending. The published Bridge recipes do not currently provide VLM pretraining.\n", "title": "gemma_4_26b_a4b_it" }, { "architecture": "ExaoneMoeForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "entries": [ { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-hf-to-megatron-cpu", "expected_result": "No CPU Hugging Face-to-Megatron conversion result is claimed. Future verification must import the complete immutable-revision checkpoint, persist iter_0000000, reload it, and audit all mapped keys, shapes, dtypes, and values.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [ { "expert_parallel": "16", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "6", "pipeline_parallel": "3", "tensor_parallel": "1", "total_gpus": 48 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 6 --gpus-per-node 8 --hf-model LGAI-EXAONE/K-EXAONE-2.0-750B-A37B --hf-revision 4bb394fd12f57be7174be7c80c30cc05472bf9e1 --megatron-path work/model-verification/k-exaone-2/imported-megatron --torch-dtype bfloat16 --tp 1 --pp 3 --ep 16 --etp 1 --distributed-timeout-minutes 180 --low-memory-save" ], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-hf-to-megatron-gpu", "expected_result": "The pinned 59,396-tensor BF16 checkpoint imported and persisted iter_0000000 on 48 H100 GPUs, and that artifact subsequently reloaded for a full distributed GPU re-export audit. The audit compared all 59,396 tensors and 1,498,715,010,176 payload bytes with zero missing, unexpected, duplicate, shape, dtype, or value mismatches.\n", "hardware": null, "last_verified": "2026-08-24", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-megatron-to-hf-cpu", "expected_result": "No CPU Megatron-to-Hugging-Face conversion result is claimed. Future verification depends on a complete CPU import and must export, strictly reload, and compare all lossless tensors against the pinned source revision.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "unverified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [ { "expert_parallel": "16", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "6", "pipeline_parallel": "3", "tensor_parallel": "1", "total_gpus": 48 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 6 --gpus-per-node 8 --hf-model LGAI-EXAONE/K-EXAONE-2.0-750B-A37B --hf-revision 4bb394fd12f57be7174be7c80c30cc05472bf9e1 --megatron-path work/model-verification/k-exaone-2/imported-megatron/iter_0000000 --hf-path work/model-verification/k-exaone-2/hf-export --torch-dtype bfloat16 --tp 1 --pp 3 --ep 16 --etp 1 --distributed-timeout-minutes 180 --distributed-save --save-every-n-ranks 1 --no-progress" ], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-megatron-to-hf-gpu", "expected_result": "A complete distributed export produced 59,396 indexed keys in 17 safetensors shards. Every exported tensor matches the pinned source bitwise across 1,498,715,008,512 payload bytes, and the corrected config/tokenizer reload successfully. The run exposed and fixed preservation of one physical repeated MTP layer across four speculative steps. The config fix is committed; this item remains unverified until the clean-commit export and exact audit rerun complete.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "unverified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "68d7e875d3bcfad5cd0e7e7d18770cc7e5630034", "command_topologies": [ { "expert_parallel": "16", "expert_tensor_parallel": "1", "pipeline_parallel": "3", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=48 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path LGAI-EXAONE/K-EXAONE-2.0-750B-A37B --hf-revision 4bb394fd12f57be7174be7c80c30cc05472bf9e1 --megatron_model_path work/model-verification/k-exaone-2/imported-megatron/iter_0000000 --hf-logits-path work/model-verification/k-exaone-2/full-forward/hf_logits_korean.pt --model_class ExaoneMoeForCausalLM --prompt \"\ub300\ud55c\ubbfc\uad6d\uc758 \uc218\ub3c4\ub294\" --tp 1 --pp 3 --ep 16 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-manual-forward-pass", "expected_result": "The memory-bounded Hugging Face reference processed all 78 decoder layers for the shared Korean prompt. Hugging Face and Megatron processed identical input IDs; the next-token matches at token 392 (literal \"?\"). Cosine similarity is 0.999993; maximum and mean absolute logit differences are 7.500000 and 1.621717.\n", "hardware": null, "last_verified": "2026-07-31", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "bd56432512613a88ceae9ecb05654b63955f6dcb", "command_topologies": [ { "expert_parallel": "16", "expert_tensor_parallel": "1", "pipeline_parallel": "3", "tensor_parallel": "1" } ], "commands": [ "uv run python scripts/inference/text_generation.py --hf-model-path LGAI-EXAONE/K-EXAONE-2.0-750B-A37B --megatron-model-path work/model-verification/k-exaone-2/imported-megatron/iter_0000000 --prompt \"\ub300\ud55c\ubbfc\uad6d\uc758 \uc218\ub3c4\ub294\" --max_new_tokens 16 --temperature 0 --top_k 1 --termination-id -1 --tp 1 --pp 3 --ep 16 --etp 1 --dtype bf16 --max_seq_length 128 --max_batch_size 1 --use-coordinator --inference-moe-token-dispatcher-type nccl --distributed-timeout-minutes 180" ], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-inference", "expected_result": "Generates an exact 16-token result. Literal completion: '?\n\nAnswer: Seoul\n\nWhat is the most pop song in the world?\n\nAnswer: \"'", "hardware": null, "last_verified": "2026-07-31", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": { "moe_dispatcher": "deepep" }, "entry_id": "k-exaone-2-pretrain-h100", "expected_result": "The exported 512-GPU H100 recipe must complete a bounded public-data pretraining run at TP4/PP16/EP32 with natural routing, finite loss, no skipped or NaN iterations, all five required metrics, and reloadable middle and final checkpoints.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-sft-h100", "expected_result": "The exported 512-GPU H100 full-SFT recipe must load the pinned checkpoint and complete a bounded immutable-dataset run at TP4/PP16/EP32 with finite loss, no skipped or NaN iterations, all four required metrics, and a reloadable final full-model checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-sft-export-inference-h100", "expected_result": "This item depends on verified full SFT. Its final checkpoint must export synchronously to Hugging Face, strictly reload, and complete deterministic greedy inference with an exact new-token count and a recorded literal completion.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-sft-long-context-h100", "expected_result": "No long-context SFT result is claimed. Future verification must first resolve a supported long-context topology, then verify sequence packing and context parallelism together with finite loss, no skipped or NaN iterations, and all five required metrics.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-peft-h100", "expected_result": "The exported 128-GPU H100 PEFT recipe must load the pinned checkpoint, verify that LoRA targets the model-native attention projections, and complete a bounded immutable-dataset run at TP4/PP8/EP16 with finite loss, no skipped or NaN iterations, all five required metrics, and a reloadable adapter checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "766f7106e6502f71f799706a1dd874644eadb9ef", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "k-exaone-2-checkpoint-resume-h100", "expected_result": "Future verification must resume directly from the pretraining middle checkpoint into a distinct output root, restore optimizer and RNG state, match the declared sentinel losses within tolerance, complete with finite metrics, and save a reloadable final checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "LGAI-EXAONE/K-EXAONE-2.0-750B-A37B", "hf_revision": "4bb394fd12f57be7174be7c80c30cc05472bf9e1", "min_transformers_version": "5.9.0", "slug": "k-exaone-2", "source_card": "examples/model_verification_cards/k-exaone-2/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. K-EXAONE 2.0 750B-A37B has public H100 recipes for pretraining, full SFT, and PEFT in this Bridge revision. CPU conversion, GPU export, training, export-after-SFT, long-context SFT, and checkpoint resume remain unverified. A complete 750B GPU import from a clean commit persisted and reloaded successfully, and a full 59,396-tensor audit is bitwise exact. A 2-GPU structural MoE/MTP proxy also completed three finite optimizer steps and saved checkpoints, but does not substitute for a full-model training item. A memory-bounded reference forward processed all 78 Hugging Face decoder layers for a Korean prompt; Megatron matched its next token with 0.999993 cosine similarity. Complete-model deterministic inference generated and independently confirmed exactly 16 new tokens. The implementation fixes are committed; GPU export remains unverified pending its own clean-commit rerun and exact audit.\n", "title": "k_exaone_2_750b_a37b" }, { "architecture": "MiniMaxM3SparseForConditionalGeneration", "base_container": "nvcr.io/nvidia/pytorch:26.04-py3", "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "entries": [ { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-hf-to-megatron-cpu", "expected_result": "No CPU HF-to-Megatron result is claimed. The CPU-only attempt enabled CPU weight initialization, but the current distributed-model initialization path still selected a CUDA device before tensor loading. With no GPU available, it exited without persisting iter_0000000, so no checkpoint reload or exact tensor audit completed. CPU conversion remains unverified rather than unsupported.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [ { "expert_parallel": "32", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model MiniMaxAI/MiniMax-M3 --hf-revision 50942730318c7943fe83db7ec8e9f9177ecb1cf8 --megatron-path work/model-verification/minimax-m3/imported-megatron --torch-dtype bfloat16 --tp 1 --pp 1 --ep 32 --etp 1 --distributed-timeout-minutes 180 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-hf-to-megatron-gpu", "expected_result": "The pinned 32-H100 full-VLM import exited successfully, persisted iter_0000000, and reloaded it at TP1/PP1/EP32/ETP1. A strict projection of the reloaded Megatron model covered exactly 23,416 of 23,416 published language, vision, projector, and lightning-indexer tensors and 854,172,958,720 actual tensor-payload bytes, with zero duplicate, missing, unexpected, shape, invalid-dtype, or value mismatches. The source index's declared total_size is 869,157,697,024 bytes. No dtype widening was required. The audit covered 1,804 total Megatron state tensors; model-state structure matched across all 32 EP ranks, and all 1,348 replicated tensors comprising 13,917,973,376 elements matched exactly.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model MiniMaxAI/MiniMax-M3 --hf-revision 50942730318c7943fe83db7ec8e9f9177ecb1cf8 --megatron-path work/model-verification/minimax-m3/imported-megatron/iter_0000000 --hf-path work/model-verification/minimax-m3/cpu-hf-export --torch-dtype bfloat16 --no-progress --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-megatron-to-hf-cpu", "expected_result": "The single-node CPU export exited successfully and wrote the complete indexed MiniMax-M3 VLM checkpoint. An exhaustive audit matched all 23,416 source and export tensors, 427,040,140,160 values, and 854,172,958,720 tensor-payload bytes with zero missing, unexpected, shape, dtype, or value mismatches and no dtype conversions (atol=0, rtol=0). A meta-device Transformers 5.14.0 reload instantiated MiniMaxM3SparseForConditionalGeneration with 1,582 state entries and zero missing, unexpected, mismatched, or error keys.\n", "hardware": null, "last_verified": "2026-08-26", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [ { "expert_parallel": "32", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model MiniMaxAI/MiniMax-M3 --hf-revision 50942730318c7943fe83db7ec8e9f9177ecb1cf8 --megatron-path work/model-verification/minimax-m3/imported-megatron/iter_0000000 --hf-path work/model-verification/minimax-m3/hf-export --torch-dtype bfloat16 --tp 1 --pp 1 --ep 32 --etp 1 --distributed-timeout-minutes 180 --distributed-save --save-every-n-ranks 1 --no-progress --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-megatron-to-hf-gpu", "expected_result": "The 32-H100 distributed export exited successfully and wrote an indexed full-VLM checkpoint at TP1/PP1/EP32/ETP1. Its key-to-shard map and all 59 safetensors shards are byte-for-byte identical to the pinned source, covering all 23,416 tensors and 854,172,958,720 actual tensor-payload bytes. The exported configuration preserves 60 text layers, 128 top-4 experts, 32 vision layers, both projector stages, and 57 lightning-indexer layers. Three tokenizer probes and multimodal processor inputs match the source. Transformers 5.14.0 natively reloads both source and export as MiniMaxM3SparseForConditionalGeneration through its checkpoint conversion mapping with zero missing, unexpected, mismatched, or error keys. The export consolidates the tokenizer in tokenizer.json and does not reproduce four redundant source-side legacy tokenizer files.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-manual-forward-pass", "expected_result": "No complete full-VLM Hugging Face/Megatron forward comparison is claimed. Future verification must process the same pinned image and prompt through both implementations, match their next-token prediction, and report cosine similarity of at least 0.99 together with maximum and mean absolute logit differences as non-gating diagnostics.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "30fae2e8796a7c78f62c9c2cf3b1e249b58fd56d", "command_topologies": [ { "expert_parallel": "32", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/inference/infer.sh --nodes 4 --gpus-per-node 8 --task vlm-generation --hf_model_path MiniMaxAI/MiniMax-M3 --hf-revision 50942730318c7943fe83db7ec8e9f9177ecb1cf8 --megatron_model_path work/model-verification/minimax-m3/imported-megatron/iter_0000000 --tp 1 --pp 1 --ep 32 --etp 1 --image_path docs/images/tp1.png --prompt \"Read the green caption near the top of the image. Answer immediately with only the exact quoted term, including capitalization and hyphen; do not explain or reason.\" --max_new_tokens 64 --trust_remote_code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-inference", "expected_result": "The 32-H100 TP1/PP1/EP32 run generated exactly 64 tokens under the 64-token maximum, with finite sampled logits and no token-0/NUL degeneration. The literal completion was \"\\u003cmm:think\\u003eThe user wants me to read the green caption near the top of the image and respond with only the exact quoted term, including capitalization and hyphen.\" A matched HybridEP run kept all observed logits finite, reached EOS, and reproduced the expected completion. Bridge-created providers and H100 recipes use flex/hybridep with 16 dispatcher SMs, unfused HybridEP permutation, and 8 ranks per H100 NVLink domain. Older checkpoints may require the same overrides after deserialization. Training remains unverified.\n", "hardware": null, "last_verified": "2026-08-08", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "minimax-m3-pretrain-h100", "expected_result": "Complete a 100-step HybridEP language-backbone run with finite loss, reported metrics, and a reloadable checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "minimax-m3-sft-h100", "expected_result": "Complete a 100-step packed HybridEP language-backbone SFT run with finite loss, reported metrics, and a reloadable checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-sft-export-inference-all", "expected_result": "The public MiniMax-M3 training recipes instantiate the language-model backbone rather than the complete VLM. Their checkpoints therefore cannot be exported as a complete Hugging Face MiniMax-M3 checkpoint containing the vision tower, projector paths, processor configuration, and lightning-indexer state. Post-SFT full-VLM export and multimodal inference remain unsupported on every hardware target.\n", "hardware": "all", "last_verified": null, "metrics": {}, "precision": null, "source_pointer": "items.sft_export_inference.all", "status": "unsupported", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "minimax-m3-sft-long-context-h100", "expected_result": "A dedicated packed long-context recipe must complete 100 steps with context parallelism, finite loss, no skipped or NaN iterations, and all five metrics. Beyond 2,048 tokens, Bridge uses full causal attention instead of MiniMax-M3's lightning-indexer sparse attention.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-peft-h100", "expected_result": "A MiniMax-M3 PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, no skipped or NaN iterations, all four metrics, and a reloadable adapter checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "6ae8e11630b212ef54f23f574f355a38f0b89d12", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "minimax-m3-checkpoint-resume-h100", "expected_result": "A direct continuation from the bounded pretrain run must restore model, optimizer, scheduler, data-order, and RNG state, execute through the reference final step, match declared sentinel losses within tolerance, and save a reloadable checkpoint to a distinct output root.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "MiniMaxAI/MiniMax-M3", "hf_revision": "50942730318c7943fe83db7ec8e9f9177ecb1cf8", "min_transformers_version": "5.14.0", "slug": "minimax-m3", "source_card": "examples/model_verification_cards/minimax-m3/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Model-level verification covers the complete MiniMax-M3 vision-language checkpoint, including the language model, vision tower, projector paths, and lightning-indexer state. The pinned 32-H100 GPU import and GPU export completed their verification gates. A current clean 32-H100 release-candidate run also reloaded the persisted full-VLM checkpoint through the public VLM generation entry point, processed the pinned repository image, and generated a coherent 64-token bounded completion without token-0 or NUL degeneration. It reached the configured bound before EOS or the image-only OCR target; that limitation is recorded in the inference result. All 59 exported safetensors shards are byte-for-byte identical to the pinned source, and Transformers 5.14.0 natively reloads the export without loading errors. A separate CPU export reproduced the complete pinned source tensor graph exactly. CPU import remains unverified because the CPU-only attempt exited during CUDA-dependent model initialization before producing a checkpoint. Full-VLM manual Hugging Face/Megatron forward correlation also remains unverified. Training entries cover only the language-model backbone and do not claim vision or projector training; post-SFT full-VLM export therefore remains unsupported.\n", "title": "minimax_m3" }, { "architecture": "KimiK3ForConditionalGeneration", "base_container": "nvcr.io/nvidia/nemo:26.06", "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "entries": [ { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-hf-to-megatron-cpu", "expected_result": "No CPU Hugging Face-to-Megatron conversion result is claimed. Future verification must import the complete pinned language backbone with the public conversion launcher, persist iter_0000000, reload it, and audit all mapped keys, shapes, dtypes, and values.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "cec012e568c55a6c0906622d8a5b0986a79f40cf", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "2", "gpus_per_node": "4", "nodes": "12", "pipeline_parallel": "3", "tensor_parallel": "2", "total_gpus": 48 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 12 --gpus-per-node 4 --hf-model moonshotai/Kimi-K3 --hf-revision 9f62e4e9fffbd0a83ddd60e1c209d828994b3569 --megatron-path work/model-verification/kimi-k3/imported-megatron --torch-dtype bfloat16 --tp 2 --pp 3 --ep 8 --etp 2 --distributed-timeout-minutes 180 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-hf-to-megatron-gpu", "expected_result": "On 48x GB200, the pinned complete checkpoint imports at TP2/PP3/EP8/ETP2 and exits successfully in 27:01. Published MXFP4 routed-expert weights are dequantized to BF16, and iter_0000000 is persisted with 48 distributed-checkpoint shards, metadata, and the recorded 93-layer run configuration. Exact Hugging Face round-trip parity is tracked separately by megatron_to_hf_gpu.\n", "hardware": null, "last_verified": "2026-07-28", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-megatron-to-hf-cpu", "expected_result": "No CPU Megatron-to-Hugging-Face conversion result is claimed. Future verification depends on a complete persisted Megatron import and must write, strictly reload, and exactly audit the exported Hugging Face language-backbone weights against the pinned source revision.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "unverified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-megatron-to-hf-gpu", "expected_result": "No GPU Megatron-to-Hugging-Face conversion result is claimed. All 69 source KDA A_log tensors were audited: each stores 96 active heads plus 32 zero-valued inactive entries, for 2,208 checked padding values. Import validation drops only this zero padding, and export unit tests restore it with zeros. A four-layer diagnostic proxy completed low-memory GPU import and distributed streaming export. Its seven safetensors shards and index, totaling 58,482,941,904 bytes, were byte-identical to an export from the previous default-save import; this validates the low-memory I/O behavior, not full-model parity. Against the published proxy source, all 16,402 keys and shapes matched and A_log padding was valid. Three o_norm tensors changed from FP32 to BF16, and 16,128 value mismatches were concentrated in MXFP4 expert packed weights and scales because import dequantizes them to BF16 before export requantizes them. Full verification therefore still requires the pinned complete model to export and strictly reload, exact comparison of lossless tensors, and a declared numerical criterion for lossy MXFP4 reconstruction.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "unverified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-manual-forward-pass", "expected_result": "A four-layer diagnostic proxy matched the next token and measured logit cosine similarity 0.9998329878 and Pearson correlation 0.9998273849. This is not a full-model comparison. Verification still requires the pinned complete language backbone on both implementations, the same prompt, a matching next token, cosine similarity of at least 0.99, and reported maximum and mean absolute logit differences.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "eb91f8be86e5f88771bfa020b33cd74dd3de80ad", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "pipeline_parallel": "3", "tensor_parallel": "1" } ], "commands": [ "uv run python examples/conversion/hf_to_megatron_generate_text.py --hf_model_path moonshotai/Kimi-K3 --trust-remote-code --apply-chat-template --thinking-mode disabled --prompt \"What is 15% of 240? Answer in one short sentence.\" --max_new_tokens 32 --tp 1 --pp 3 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-inference", "expected_result": "On 24x GB300, the pinned complete language backbone imports directly into Megatron at TP1/PP3/EP8/ETP1. The command exits successfully in 22:59 after exactly 32 new greedy tokens and prints this completion, represented with JSON Unicode escapes for the model control-token brackets: \"15% of 240 = 0.15 \u00d7 240 = 36.\\u003c|close|\\u003ethink\\u003c|sep|\\u003e\\u003c|open|\\u003eresponse\\u003c|sep|\\u003e15% of 240 is 36.\".", "hardware": null, "last_verified": "2026-07-27", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-pretrain-gb300", "expected_result": "Training was intentionally excluded from this model-only verification pass. A future public Kimi K3 language-backbone recipe must complete a bounded 100-step pretrain run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.GB300", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-sft-gb300", "expected_result": "Training was intentionally excluded from this model-only verification pass. A future public Kimi K3 language-backbone recipe must complete a pinned-data 100-step full-SFT run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.GB300", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-sft-export-inference-gb300", "expected_result": "No post-SFT export or inference result is claimed because SFT was intentionally excluded from this model-only verification pass. Future verification requires a completed SFT run followed by a strict Hugging Face export reload and deterministic greedy text generation.\n", "hardware": "GB300", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB300", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-sft-long-context-gb300", "expected_result": "Training was intentionally excluded from this model-only verification pass. A future public long-context SFT recipe must define supported sequence packing and context parallelism, complete 100 steps with finite loss and no skipped or NaN iterations, and record all five metrics.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.GB300", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-peft-gb300", "expected_result": "Training was intentionally excluded from this model-only verification pass. A future Kimi K3 PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.GB300", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "ae77a6703c61669f15936a12ddf542c2cf5cbc3b", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "kimi-k3-checkpoint-resume-gb300", "expected_result": "Training was intentionally excluded from this model-only verification pass. A future direct continuation must restore model, optimizer, scheduler, data-order, and RNG state, match declared sentinel losses within tolerance, and save a reloadable checkpoint to a distinct output root.\n", "hardware": "GB300", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.GB300", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "moonshotai/Kimi-K3", "hf_revision": "9f62e4e9fffbd0a83ddd60e1c209d828994b3569", "min_transformers_version": "4.56.2", "slug": "kimi-k3", "source_card": "examples/model_verification_cards/kimi-k3/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. This card covers the Kimi K3 language backbone in the published multimodal checkpoint. It does not claim support for the vision tower or multimodal inputs. Full-checkpoint GPU import and deterministic Megatron greedy text generation are verified. CPU conversion, GPU export, full-model Hugging Face/Megatron forward correlation, and every training workflow remain unverified.\n", "title": "kimi_k3" }, { "architecture": "DeepseekV3ForCausalLM", "base_container": "nvcr.io/nvidia/pytorch:26.04-py3", "bridge_commit": "93c6930597fd3766acd177288b7a8607d7b96ad7", "entries": [ { "bridge_commit": "93c6930597fd3766acd177288b7a8607d7b96ad7", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model moonshotai/Moonlight-16B-A3B --megatron-path work/model-verification/moonlight-16b-a3b/cpu-megatron --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-hf-to-megatron-cpu", "expected_result": "The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with exact model configuration. After native reload, all 377 registered parameters and persistent buffers match bitwise by key, shape, dtype, and value. In the serialized files, 26 router correction-bias buffers widen losslessly from BF16 to FP32.\n", "hardware": null, "last_verified": "2026-07-19", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "93c6930597fd3766acd177288b7a8607d7b96ad7", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model moonshotai/Moonlight-16B-A3B --megatron-path work/model-verification/moonlight-16b-a3b/imported-megatron --torch-dtype bfloat16 --tp 1 --pp 1 --ep 8 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-hf-to-megatron-gpu", "expected_result": "The command exits successfully, creates a reloadable iter_0000000, and an exact audit finds all 5,344 serialized source tensors and the model configuration unchanged after the paired GPU export.\n", "hardware": null, "last_verified": "2026-07-19", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "93c6930597fd3766acd177288b7a8607d7b96ad7", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model moonshotai/Moonlight-16B-A3B --megatron-path work/model-verification/moonlight-16b-a3b/cpu-megatron/iter_0000000 --hf-path work/model-verification/moonlight-16b-a3b/cpu-hf-export --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-megatron-to-hf-cpu", "expected_result": "The command exits successfully, the model configuration is exact, and all 377 registered parameters and persistent buffers match bitwise by key, shape, dtype, and value after native reload. Transformers strictly reloads the output as DeepseekV3ForCausalLM with trust_remote_code disabled. In the serialized files, 26 router correction-bias buffers widen losslessly from BF16 to FP32 and 27 stale, nonpersistent source inv_freq tensors are omitted.\n", "hardware": null, "last_verified": "2026-07-19", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "93c6930597fd3766acd177288b7a8607d7b96ad7", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model moonshotai/Moonlight-16B-A3B --megatron-path work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --hf-path work/model-verification/moonlight-16b-a3b/hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 1 --pp 1 --ep 8 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-megatron-to-hf-gpu", "expected_result": "Strict distributed export exits successfully, all 5,344 serialized tensor keys, shapes, dtypes, and values match the recorded HF revision, and native Transformers strictly reloads the output as DeepseekV3ForCausalLM with trust_remote_code disabled and no missing or unexpected keys.\n", "hardware": null, "last_verified": "2026-07-19", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "056616080f2d8202bfe3f636069584406b5e465e", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "pipeline_parallel": "1", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path moonshotai/Moonlight-16B-A3B --megatron_model_path work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --tp 1 --pp 1 --ep 8 --etp 1 --prompt \"The capital of France is\" --model_class DeepseekV3ForCausalLM --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-manual-forward-pass", "expected_result": "Hugging Face and Megatron have next-token matches at token ID 17374 (\" Paris\"). Cosine similarity is 0.999893, above the 0.99 correlation gate. The maximum absolute logit difference is 0.472656 and the mean absolute logit difference is 0.074414; both are report-only diagnostic observations. This historical result predates explicit helper revision pinning and is retained against the recorded immutable HF revision.\n", "hardware": null, "last_verified": "2026-07-19", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "93c6930597fd3766acd177288b7a8607d7b96ad7", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "pipeline_parallel": "1", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/hf_to_megatron_generate_text.py --hf_model_path moonshotai/Moonlight-16B-A3B --megatron_model_path work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --tp 1 --pp 1 --ep 8 --etp 1 --prompt \"The capital of France is\" --max_new_tokens 32 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-inference", "expected_result": "Two independent executions using greedy decoding each produce exactly 32 new tokens with byte-identical token IDs and this literal completion, including its leading space, \" Paris. It is the largest city in France and is known for its iconic landmarks such as the Eiffel Tower, the Louvre Museum, and Notre-Dame\".\n", "hardware": null, "last_verified": "2026-07-19", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "411c6b1dc26eef50124843fbf9b6253b7b6c43cb", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "moonlight_16b_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe moonlight_16b_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/moonlight-16b-a3b/rp2-convergence-v1 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load=null dist.distributed_timeout_minutes=30 --save_dir work/model-verification/moonlight-16b-a3b/pretrain-convergence-v1-reference-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "moonlight-16b-a3b-pretrain-h100", "expected_result": "The uninterrupted 16-GPU bounded RP2 run completes exactly 100 steps at TP1/PP1/CP1/EP8/ETP1, DP16, GBS/MBS 1024/2, and 32-way gradient accumulation with natural MoE routing and the HybridEP flex dispatcher. It reaches peak learning rate at step 40 and completes cosine decay at step 100. LM loss is finite from 12.43248 to 5.158420 with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved.\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": { "final_loss": 5.15842, "initial_loss": 12.43248, "last_10_steps_model_tflops_per_gpu_avg": 386.59, "last_10_steps_step_time_ms_avg": 11636.35, "last_10_steps_tokens_per_second_per_gpu_avg": 22528.026 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "4b8a6929348d6e25e32744d89228254cc3985564", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "moonlight_16b_sft_8gpu_h100_bf16_tp1_config", "sequence_length": "8192", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe moonlight_16b_sft_8gpu_h100_bf16_tp1_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"476b36a473d4467f94469414bef6cee75c9c8172\"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-sft-hybridep-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 'dataset.dataset_kwargs={pad_to_max_length:true}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/moonlight-16b-a3b/sft-convergence-hybridep-v3-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "moonlight-16b-a3b-sft-h100", "expected_result": "The immutable-revision 8-GPU run completes exactly 100 full-SFT steps at TP1/PP1/CP1/EP8/ETP1, DP8, SP off, GBS/MBS 8/1, and no gradient accumulation, using the HybridEP flex dispatcher with overlapped expert communication and delayed weight-gradient compute. Fixed-width 8K, pad-1 offline packing is 99.73% efficient and averages 21.550 source sequences per pack while retaining 6,553,600 token slots across the 100 updates. LM loss is finite from 1.770799 to 1.180197 with no skipped or NaN iterations, all five metrics are recorded, and the complete eight-shard iter_0000100 full-model checkpoint saves successfully.\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": { "final_loss": 1.180197, "initial_loss": 1.770799, "last_10_steps_model_tflops_per_gpu_avg": 229.5, "last_10_steps_step_time_ms_avg": 562.14, "last_10_steps_tokens_per_second_per_gpu_avg": 14572.882 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "4b8a6929348d6e25e32744d89228254cc3985564", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model moonshotai/Moonlight-16B-A3B --megatron-path work/model-verification/moonlight-16b-a3b/sft-convergence-hybridep-v3-checkpoints/iter_0000100 --hf-path work/model-verification/moonlight-16b-a3b/sft-convergence-hybridep-v3-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 1 --pp 1 --ep 8 --etp 1 --trust-remote-code", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/moonlight-16b-a3b/sft-convergence-hybridep-v3-hf-export --prompt \"In one short sentence, explain why Paris is important to France.\" --max-new-tokens 32 --chat-template" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-sft-export-inference-h100", "expected_result": "The immutable-revision step-100 SFT checkpoint exports as 27 indexed BF16 shards with 5,344 weights, vocabulary size 163842, and embedding and output-head shapes [163842, 2048]. The index has the exact key set and total_size 31,920,240,256, equal to the serialized tensor bytes. Transformers strictly reloads with no errors, mismatches, missing, or unexpected keys. Two independent runs using greedy decoding produce byte-identical token IDs and exactly 32 new tokens with this literal completion: \"Paris is the capital city of France and is considered the cultural, political, and economic center of the country. It is home to many of France's most famous\".\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "f3ae2767b5e18aeb67b726cd8d5f1db58216dcc9", "command_topologies": [ { "context_parallel": "2", "gpus_per_node": "8", "nodes": "1", "recipe": "moonlight_16b_sft_8k_config", "sequence_length": "8192", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe moonlight_16b_sft_8k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 8192 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"476b36a473d4467f94469414bef6cee75c9c8172\"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "offline" }, "entry_id": "moonlight-16b-a3b-sft-long-context-h100", "expected_result": "The immutable-revision 8-GPU run completes exactly 20 Tulu3 SFT steps at the model's 8192-token context limit with recipe-owned TP2/PP1/CP2/EP8/SP-on, GBS/MBS 128/1, and explicit pad-4 offline packing. LM loss is 1.306440 to 1.228019; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.69%, and the sampled training window contains 13,765,732 actual supervised tokens.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 1.228019, "initial_loss": 1.30644, "last_10_steps_model_tflops_per_gpu_avg": 40.84, "last_10_steps_step_time_ms_avg": 60599.31, "last_10_steps_tokens_per_second_per_gpu_avg": 2162.929 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "411c6b1dc26eef50124843fbf9b6253b7b6c43cb", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "recipe": "moonlight_16b_peft_4gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 4 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe moonlight_16b_peft_4gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"476b36a473d4467f94469414bef6cee75c9c8172\"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/moonlight-16b-a3b/peft-convergence-v1-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "moonlight-16b-a3b-peft-h100", "expected_result": "The immutable-revision 4-GPU run completes exactly 100 PEFT steps at TP1/PP1/CP1/EP4/ETP1, DP4, SP off, GBS/MBS 32/1, and eight-way gradient accumulation. Pad-4 offline packing is 99.29% efficient, and the sampled 6,553,600 token slots contain 4,291,091 supervised tokens after label masking. Only rank-8, alpha-16, zero-dropout LoRA on linear_q_proj, linear_kv_down_proj, linear_kv_up_proj, and linear_proj is trainable. LM loss is finite from 1.264324 to 1.100755 with no skipped or NaN iterations, all five metrics are recorded, and the complete four-shard iter_0000100 adapter checkpoint covers all 216 expected adapter entries.\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": { "final_loss": 1.100755, "initial_loss": 1.264324, "last_10_steps_model_tflops_per_gpu_avg": 67.38, "last_10_steps_step_time_ms_avg": 3822.84, "last_10_steps_tokens_per_second_per_gpu_avg": 4285.819 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "411c6b1dc26eef50124843fbf9b6253b7b6c43cb", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "moonlight_16b_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe moonlight_16b_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/moonlight-16b-a3b/rp2-convergence-v1 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.finetune=false dist.distributed_timeout_minutes=30 --load_dir work/model-verification/moonlight-16b-a3b/pretrain-convergence-v1-reference-checkpoints --save_dir work/model-verification/moonlight-16b-a3b/pretrain-convergence-v1-resumed-checkpoints --save_interval 50 checkpoint.ckpt_step=50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "moonlight-16b-a3b-checkpoint-resume-h100", "expected_result": "The direct 16-GPU continuation restores optimizer, scheduler, data-order, and RNG state from iter_0000050, begins at step 51, and finishes exactly at step 100 in a distinct output root at TP1/PP1/CP1/EP8/ETP1, DP16, GBS/MBS 1024/2, and 32-way gradient accumulation. All 50 losses are finite with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and a complete 16-shard iter_0000100 checkpoint is saved. Step-51 resumed/reference loss is 5.775853/5.775853. Step-100 resumed/reference loss is 5.150805/5.158420, an absolute difference of 0.007615 that satisfies the declared 1e-6-plus-1%-relative tolerance.\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": { "final_loss": 5.150805, "initial_loss": 5.775853, "last_10_steps_model_tflops_per_gpu_avg": 383.92, "last_10_steps_step_time_ms_avg": 11717.96, "last_10_steps_tokens_per_second_per_gpu_avg": 22371.129 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "moonshotai/Moonlight-16B-A3B", "hf_revision": "476b36a473d4467f94469414bef6cee75c9c8172", "min_transformers_version": "5.8.1", "slug": "moonlight-16b-a3b", "source_card": "examples/model_verification_cards/moonlight-16b-a3b/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Moonlight-16B-A3B support verification covers conversion, inference, and training.\n", "title": "moonlight_16b_a3b" }, { "architecture": "NemotronH_Nano_Omni_Reasoning_V3", "base_container": "nvcr.io/nvidia/nemo:26.06", "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "entries": [ { "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "command_topologies": [ { "expert_parallel": "1", "expert_tensor_parallel": "1", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 --hf-revision 24e67ea000b7c2837fc8f9488aa2008524fac8ba --megatron-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/cpu-megatron-clean --torch-dtype bfloat16 --tp 1 --pp 1 --ep 1 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-hf-to-megatron-cpu", "expected_result": "The offline, immutable-revision CPU import exits successfully after mapping 7,333 model parameters and creates a reloadable iter_0000000 torch_dist checkpoint with 33,015,546,816 parameters on the single model parallel rank.\n", "hardware": null, "last_verified": "2026-07-30", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "command_topologies": [ { "expert_parallel": "4", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 --hf-revision 24e67ea000b7c2837fc8f9488aa2008524fac8ba --megatron-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean --torch-dtype bfloat16 --tp 2 --pp 1 --ep 4 --etp 1 --trust-remote-code --low-memory-save" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-hf-to-megatron-gpu", "expected_result": "The command exits successfully at TP2/PP1/EP4/ETP1 and creates a reloadable iter_0000000 checkpoint. The paired strict GPU export contains the source checkpoint's exact 7,349-key set, shapes, dtypes, and values.\n", "hardware": null, "last_verified": "2026-07-30", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "7783fc356b7d4768525eabf9825f9108c6544f4c", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 --hf-revision 24e67ea000b7c2837fc8f9488aa2008524fac8ba --megatron-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/cpu-megatron-clean/iter_0000000 --hf-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/cpu-hf-export-clean --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-megatron-to-hf-cpu", "expected_result": "CPU export completes in 14 indexed shards totaling 66,031,270,520 bytes. Its exact comparison contains all 7,349 source tensors and 33,015,632,238 values, with identical keys, shapes, dtypes, and values and no dtype conversions. Transformers reloads the local custom-code export natively with the source-equivalent warning set: one generated vision summary buffer is missing and four source-only audio/vision buffers are unexpected.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "7783fc356b7d4768525eabf9825f9108c6544f4c", "command_topologies": [ { "expert_parallel": "4", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 --hf-revision 24e67ea000b7c2837fc8f9488aa2008524fac8ba --megatron-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --hf-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-hf-export-clean --torch-dtype bfloat16 --tp 2 --pp 1 --ep 4 --etp 1 --trust-remote-code --distributed-save --save-every-n-ranks 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-megatron-to-hf-gpu", "expected_result": "Distributed export completes in 17 indexed shards totaling 66,031,270,520 bytes. All 7,349 tensors and 33,015,632,238 values match the immutable HF source exactly in keys, shapes, dtypes, and values with no dtype conversions. Transformers reloads the local custom-code export natively with the same one missing generated buffer and four unexpected source-only buffers reported when reloading the pinned source itself.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "command_topologies": [ { "expert_parallel": "2", "expert_tensor_parallel": "1", "pipeline_parallel": "4", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 --hf-revision 24e67ea000b7c2837fc8f9488aa2008524fac8ba --megatron_model_path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --tp 1 --pp 4 --ep 2 --etp 1 --prompt \"The capital of France is \" --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-manual-forward-pass", "expected_result": "The pinned one-step run exits successfully and both implementations predict token ID 6993 (\" Paris\"), but cosine similarity is 0.969760 and therefore does not satisfy the required 0.99 verification gate. The maximum and mean absolute logit differences are 3.527344 and 0.527820, respectively.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "command_topologies": [ { "expert_parallel": "2", "expert_tensor_parallel": "1", "pipeline_parallel": "4", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/models/nemotron/nemotron_3_omni/hf_to_megatron_generate_nemotron_omni.py --hf_model_path nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 --hf-revision 24e67ea000b7c2837fc8f9488aa2008524fac8ba --megatron_model_path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --prompt \"The capital of France is\" --max_new_tokens 4 --tp 1 --pp 4 --ep 2 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-inference", "expected_result": "Deterministic greedy decoding returns the exact 4-token result and\nterminates with the model end token. Literal completion: \"\nParis.\"", "hardware": null, "last_verified": "2026-07-30", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-pretrain-all", "expected_result": "Megatron Bridge does not publish a pretraining recipe for this multimodal conditional-generation model; the supported package provides supervised CORD v2 SFT and LoRA PEFT workflows.\n", "hardware": "all", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.pretrain.all", "status": "unsupported", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "9b69b47a321de14b0453c230dd130e18ae0f00f7", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_omni_cord_v2_sft_8gpu_h100_bf16_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_sft_8gpu_h100_bf16_config --mode sft --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-4k-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:\"7f0115a4b758a71d6473b8d085751692da2fef98\"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-sft-h100", "expected_result": "The immutable-revision, real image-text CORD v2 run completes exactly 100 full-SFT optimizer steps on eight H100 GPUs at TP2/PP1/CP1/EP8/ETP1 and GBS/MBS 64/4 with natural-routing HybridEP. Execution uses selective MoE/layernorm recompute, fused attention, TE cross entropy, fused weighted squared-ReLU, and router fusion without changing the image-text objective or routing policy. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.281308 to 0.03330838, the final ten steps average 3,603.67 ms and 197.10 TFLOP/s/GPU, sampled peak memory is 76,349 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 at 197.07 final-ten TFLOP/s/GPU with finite loss and zero skipped or NaN iterations.\n", "hardware": "H100", "last_verified": "2026-08-12", "metrics": { "final_loss": 0.03330838, "initial_loss": 1.281308, "last_10_steps_model_tflops_per_gpu_avg": 197.1, "last_10_steps_step_time_ms_avg": 3603.67, "last_10_steps_tokens_per_second_per_gpu_avg": 9092.952 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "2", "total_gpus": 8 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 --hf-revision 24e67ea000b7c2837fc8f9488aa2008524fac8ba --megatron-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-4k-checkpoints/iter_0000100 --hf-path work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-hf-export-clean --torch-dtype bfloat16 --tp 2 --pp 1 --ep 8 --etp 1 --trust-remote-code --distributed-save --not-strict", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-hf-export-clean --prompt \"The capital of France is\" --max-new-tokens 32 --chat-template --disable-thinking --trust-remote-code --device cuda --dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-sft-export-inference-h100", "expected_result": "Non-strict export succeeds and writes 6,637 indexed tensors in 17 shards. The CORD v2 recipe intentionally disables sound, so 712 source audio tensors are absent. Native Transformers generation remains unverified because local dynamic-code loading omits configuration_nemotron_h before model weights can reload.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "eb7cf72392ac07e7a095b87d9b1d6bf2b38c5b6e", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_omni_cord_v2_long_context_sft_8gpu_h100_bf16_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_long_context_sft_8gpu_h100_bf16_config --mode sft --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-8k-packed-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:\"7f0115a4b758a71d6473b8d085751692da2fef98\"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "in_batch" }, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-sft-long-context-h100", "expected_result": "The immutable-revision, real image-text CORD v2 run completes exactly 100 full-SFT optimizer steps on eight H100 GPUs at TP4/PP1/CP2/EP8/ETP1 and GBS/MBS 64/2 with 8K in-batch packing and the standard all-to-all dispatcher. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.251364 to 0.01020405, the final ten steps average 15,577.9 ms and 93.73 TFLOP/s/GPU, sampled peak memory is 75,197 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 with finite loss and zero skipped or NaN iterations.\n", "hardware": "H100", "last_verified": "2026-08-11", "metrics": { "final_loss": 0.01020405, "initial_loss": 1.251364, "last_10_steps_model_tflops_per_gpu_avg": 93.73, "last_10_steps_step_time_ms_avg": 15577.9, "last_10_steps_tokens_per_second_per_gpu_avg": 4206.986 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "eb7cf72392ac07e7a095b87d9b1d6bf2b38c5b6e", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_omni_cord_v2_peft_8gpu_h100_bf16_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_peft_8gpu_h100_bf16_config --mode lora --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/peft-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:\"7f0115a4b758a71d6473b8d085751692da2fef98\"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-peft-h100", "expected_result": "The immutable-revision, real image-text CORD v2 run completes exactly 100 LoRA optimizer steps on eight H100 GPUs at TP2/PP1/CP1/EP8/ETP1 and GBS/MBS 64/1 with natural-routing HybridEP. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.282941 to 0.02634745, the final ten steps average 11,621.42 ms and 61.25 TFLOP/s/GPU, sampled peak memory is 44,873 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 with finite loss and zero skipped or NaN iterations.\n", "hardware": "H100", "last_verified": "2026-08-11", "metrics": { "final_loss": 0.02634745, "initial_loss": 1.282941, "last_10_steps_model_tflops_per_gpu_avg": 61.25, "last_10_steps_step_time_ms_avg": 11621.42, "last_10_steps_tokens_per_second_per_gpu_avg": 2819.621 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "fbbafc7ddfa818ef91d5eb64c8a00dcaa7bee78d", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-omni-30b-a3b-reasoning-checkpoint-resume-all", "expected_result": "There is no supported pretraining reference workflow for this model, so no model-wide optimizer and RNG checkpoint-resume contract is available for sentinel comparison.\n", "hardware": "all", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.checkpoint_resume.all", "status": "unsupported", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "hf_revision": "24e67ea000b7c2837fc8f9488aa2008524fac8ba", "min_transformers_version": "5.8.0", "slug": "nemotron-3-nano-omni-30b-a3b-reasoning", "source_card": "examples/model_verification_cards/nemotron-3-nano-omni-30b-a3b-reasoning/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Verification uses the immutable public model and CORD v2 revisions. CPU and distributed GPU import and export, deterministic Megatron inference, 100-step one-node H100 full-model SFT, in-batch-packed 8K SFT, and LoRA PEFT runs completed. The H100 fine-tuning variants preserve real image-text samples and natural routing while using lower-precision optimizer state; 4K SFT and PEFT use HybridEP, while packed CP2 uses the standard all-to-all dispatcher. Strict CPU and GPU round trips preserve all 7,349 tensors bitwise and reload with the same native-loader notices as the pinned Hugging Face source. The one-step HF/Megatron comparison predicts the same token but remains below the 0.99 cosine gate. Unsupported and incomplete workflows remain explicitly identified rather than inferred from focused unit coverage.\n", "title": "nemotron_3_nano_omni_30b_a3b_reasoning" }, { "architecture": "NemotronHForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "entries": [ { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-hf-to-megatron-cpu", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --hf-revision 2d59de1cbd51c0adf384eb906b766d1aee0e0517 --megatron-path work/model-verification/nemotron-3-nano-30b-a3b/import-gpu --torch-dtype bfloat16 --tp 1 --pp 1 --ep 8 --etp 1 --distributed-timeout-minutes 120 --trust-remote-code --low-memory-save" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-hf-to-megatron-gpu", "expected_result": "The pinned one-node 8-H100 import exits successfully at TP1/PP1/EP8/ETP1, persists iter_0000000, and reloads it for inference. A separate public GPU round-trip at the same topology exhaustively compares all 6,243 exported BF16 weights against the immutable source: 6,243 of 6,243 match with no skipped FP8 tensors or mismatches.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --hf-revision 2d59de1cbd51c0adf384eb906b766d1aee0e0517 --megatron-path work/model-verification/nemotron-3-nano-30b-a3b/import-gpu/iter_0000000 --hf-path work/model-verification/nemotron-3-nano-30b-a3b/cpu-hf-export --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-megatron-to-hf-cpu", "expected_result": "The synchronous CPU export writes a complete indexed checkpoint. All 6,243 source keys and shapes match across 31,577,940,288 values. The 46 source FP32 tensors cast to BF16 are the 23 per-layer mixer A_log tensors and 23 per-layer mixer D tensors; each equals the export exactly after that declared cast, and every remaining tensor matches exactly at zero tolerance. Native loading succeeds as NemotronHForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --hf-revision 2d59de1cbd51c0adf384eb906b766d1aee0e0517 --megatron-path work/model-verification/nemotron-3-nano-30b-a3b/import-gpu/iter_0000000 --hf-path work/model-verification/nemotron-3-nano-30b-a3b/gpu-hf-export --torch-dtype bfloat16 --tp 1 --pp 1 --ep 8 --etp 1 --distributed-save --save-every-n-ranks 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-megatron-to-hf-gpu", "expected_result": "The synchronous TP1/PP1/EP8/ETP1 export writes a complete indexed checkpoint. All 6,243 source keys and shapes match across 31,577,940,288 values. The 69 source FP32 tensors cast to BF16 are the same 46 mixer A_log and D tensors plus 23 per-layer mixer gate e_score_correction_bias tensors. The legacy distributed GPU export model wrapper downcast those gate biases to BF16, while the CPU export preserved them in FP32; all 69 equal the export exactly after this declared cast, and every remaining tensor matches exactly at zero tolerance. Native loading succeeds as NemotronHForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-manual-forward-pass", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 8 --task legacy-full-prefix-generation --hf_model_path nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --hf-revision 2d59de1cbd51c0adf384eb906b766d1aee0e0517 --megatron_model_path work/model-verification/nemotron-3-nano-30b-a3b/import-gpu/iter_0000000 --tp 1 --pp 1 --ep 8 --etp 1 --prompt \"Explain why the sky appears blue in one sentence.\" --max_new_tokens 32 --apply-chat-template --thinking-mode disabled --trust-remote-code --legacy-full-prefix" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-inference", "expected_result": "The public launcher reloaded the pinned BF16 checkpoint and produced exactly 32 generated tokens across generation steps 0 through 31, using the 32-token maximum without an earlier EOS. The literal completion is 'User asks: \"Explain why the sky appears blue in one sentence.\" So we need to give a single sentence explanation. Provide concise answer. Probably: \"'.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-pretrain-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-sft-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-sft-export-inference-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-sft-long-context-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-peft-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-checkpoint-resume-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "0480586879f2513958fd8634fc529693ae13e536", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_nano_pretrain_8gpu_gb300_fp8mx_config", "sequence_length": "8192", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 4 --recipe nemotron_3_nano_pretrain_8gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 8192 logger.save_config_filepath=work/model-verification/nemotron-3-nano-30b-a3b/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-30b-a3b-pretrain-performance-gb300", "expected_result": "On exactly 8 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP8/ETP1, GBS/MBS 512/4, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.18025 to 0.004964447; the final ten steps average 12455.140 ms, 937.770 TFLOP/s/GPU, and 42094.107 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB300", "last_verified": "2026-08-18", "metrics": { "final_loss": 0.004964447, "initial_loss": 12.18025, "last_10_steps_model_tflops_per_gpu_avg": 937.77, "last_10_steps_step_time_ms_avg": 12455.14, "last_10_steps_tokens_per_second_per_gpu_avg": 42094.107 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "hf_revision": "2d59de1cbd51c0adf384eb906b766d1aee0e0517", "min_transformers_version": "5.8.1", "slug": "nemotron-3-nano-30b-a3b", "source_card": "examples/model_verification_cards/nemotron-3-nano-30b-a3b/card.yaml", "summary": "The pinned checkpoint has verified one-node GPU import, independent CPU and distributed GPU BF16 export, and checkpoint-backed deterministic inference. Both exports strictly reload with Transformers and exhaustively match the source after the explicitly requested BF16 casts. Performance scope: pretrain_performance.GB300 uses the tuned canonical 8-GPU MXFP8 recipe. CPU import, manual forward, and functional training workflows remain unverified in this card.\n", "title": "nemotron_3_nano_30b_a3b" }, { "architecture": "NemotronHForCausalLM", "base_container": "nvcr.io/nvidia/pytorch:26.04-py3", "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "entries": [ { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 --hf-revision dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f --megatron-path work/model-verification/nemotron-3-nano-4b/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-hf-to-megatron-cpu", "expected_result": "The command exits successfully and creates iter_0000000. After the paired CPU export, all 263 tensors and 3,973,556,832 parameters match the pinned source exactly in name, shape, dtype, and value, with maximum difference zero.\n", "hardware": null, "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "tensor_parallel": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 1 --tp 1 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 --hf-revision dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f --megatron-path work/model-verification/nemotron-3-nano-4b/imported-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-hf-to-megatron-gpu", "expected_result": "The command exits successfully and creates a reloadable iter_0000000. After the paired GPU export, all 263 tensors and 3,973,556,832 parameters match the pinned source exactly in name, shape, dtype, and value.\n", "hardware": null, "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 --hf-revision dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f --megatron-path work/model-verification/nemotron-3-nano-4b/cpu-megatron --hf-path work/model-verification/nemotron-3-nano-4b/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-megatron-to-hf-cpu", "expected_result": "Strict export exits successfully; all 263 BF16 tensors match the pinned source bitwise, the 171-byte generation configuration is preserved byte-for-byte, and Transformers reloads the output natively as NemotronHForCausalLM.\n", "hardware": null, "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "tensor_parallel": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 1 --tp 1 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 --hf-revision dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f --megatron-path work/model-verification/nemotron-3-nano-4b/imported-megatron --hf-path work/model-verification/nemotron-3-nano-4b/hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-megatron-to-hf-gpu", "expected_result": "Strict export exits successfully; all 263 BF16 tensors match the pinned source bitwise, the generation configuration is exact, and Transformers reloads the output natively as NemotronHForCausalLM. A_log, D, dt_bias, and out_proj are exact for all 21 Mamba layers.\n", "hardware": null, "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=1 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 --hf-revision dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f --megatron_model_path work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --tp 1 --prompt \"The capital of France is the city of\"" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-manual-forward-pass", "expected_result": "The eight-token one-step comparison exits successfully using native Transformers Nemotron-H and a real Megatron inference context. Both paths have next-token predictions that match at token ID 6993 (\" Paris\"); cosine similarity is 0.999982 and the maximum and mean absolute logit differences are 0.093750 and 0.018522.\n", "hardware": null, "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "78ecfad7644ee67efa4e8af4b0e7cf09f6bc9ecc", "command_topologies": [ { "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=1 examples/conversion/hf_to_megatron_generate_text.py --hf_model_path nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 --hf-revision dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f --megatron_model_path work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --tp 1 --prompt \"The capital of France is\" --max_new_tokens 2" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-inference", "expected_result": "Two independent runs use greedy decoding, produce byte-identical token IDs, and return exactly 2 generated tokens with this literal completion, including its leading space: ' Paris.\"'.\n", "hardware": null, "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_3_nano_4b_pretrain_8gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_pretrain_8gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/nemotron-3-nano-4b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 dist.distributed_timeout_minutes=30 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/pretrain-reference-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-pretrain-h100", "expected_result": "The uninterrupted bounded random-initialization RP2 reference finishes all 100 steps at recipe-owned GBS/MBS 1024/1 with the five recorded metrics, finite loss, and no skipped or NaN iterations. It reaches peak learning rate 3e-4 at step 40 and completes cosine decay to 3e-5 at step 100. Both iter_0000050 and iter_0000100 contain all eight distributed shards plus metadata, optimizer/RNG train state, run config, and tokenizer; iter_0000100 reloads at step 100 without an extra optimizer step.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 5.511951, "initial_loss": 12.4325, "last_10_steps_model_tflops_per_gpu_avg": 402.51, "last_10_steps_step_time_ms_avg": 26775.11, "last_10_steps_tokens_per_second_per_gpu_avg": 19581.171 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_3_nano_4b_sft_8gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:1}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "nemotron-3-nano-4b-sft-h100", "expected_result": "Pad-1 offline packing is 99.27% efficient. The TP1/DP8 run uses four microbatches per optimizer step and completes all 100 steps with the four recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, assistant-only loss masks contain 4,187,630 supervised tokens. The complete eight-shard iter_0000100 full-model checkpoint is saved and reloads for export.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 1.234906, "initial_loss": 1.839282, "last_10_steps_model_tflops_per_gpu_avg": 262.89, "last_10_steps_step_time_ms_avg": 627.0, "last_10_steps_tokens_per_second_per_gpu_avg": 13065.391 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "tensor_parallel": "1", "total_gpus": 1 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 1 --tp 1 --hf-model nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 --hf-revision dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f --megatron-path work/model-verification/nemotron-3-nano-4b/sft-checkpoints/iter_0000100 --hf-path work/model-verification/nemotron-3-nano-4b/sft-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/nemotron-3-nano-4b/sft-hf-export --prompt \"Name the capital of France and explain its role in one sentence.\" --max-new-tokens 45 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-sft-export-inference-h100", "expected_result": "The final full-SFT checkpoint exports as one BF16 safetensors file with 263 weights and reloads natively as NemotronHForCausalLM. The helper runs greedy generation twice, obtains identical token IDs and exactly 45 new tokens, and produces this literal completion with twelve trailing newline characters: \"The capital of France is Paris, which serves as the country's political, cultural, and economic center.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\".\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "context_parallel": "2", "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_3_nano_4b_sft_8gpu_h100_bf16_32k_config", "sequence_length": "32768", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_sft_8gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 32768 --tensor_model_parallel_size 2 --context_parallel_size 2 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-long-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.calculate_per_token_loss=true ddp.average_in_collective=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/long-sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "offline" }, "entry_id": "nemotron-3-nano-4b-sft-long-context-h100", "expected_result": "Pad-4 offline packing is 99.35% efficient. The TP2/CP2/SP-on/DP2 run uses four microbatches per optimizer step and completes all 100 32K steps with the five recorded metrics, finite loss, and no skipped or NaN iterations. Across 26,214,400 token slots, assistant-only loss masks contain 16,734,347 supervised tokens. The complete eight-shard iter_0000100 checkpoint is saved and reloads at the same topology.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 1.100197, "initial_loss": 1.65866, "last_10_steps_model_tflops_per_gpu_avg": 142.06, "last_10_steps_step_time_ms_avg": 5575.59, "last_10_steps_tokens_per_second_per_gpu_avg": 5877.046 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_3_nano_4b_peft_8gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_peft_8gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:4}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "nemotron-3-nano-4b-peft-h100", "expected_result": "Pad-4 offline packing is 99.36% efficient. The frozen-base LoRA run uses rank 8, alpha 16, zero dropout, and linear_qkv/linear_proj targets. Its TP1/DP8 layout uses four microbatches per optimizer step and completes all 100 steps with finite loss and no skipped or NaN iterations. Across 6,553,600 token slots, assistant-only loss masks contain 4,178,503 supervised tokens. The complete eight-shard iter_0000100 adapter checkpoint reloads over the pinned base model at step 100.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 1.256916, "initial_loss": 1.769892, "last_10_steps_model_tflops_per_gpu_avg": 431.9, "last_10_steps_step_time_ms_avg": 381.69, "last_10_steps_tokens_per_second_per_gpu_avg": 21462.443 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "0e6c1837edeb442b07c9175b855b3d288c5cad33", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_3_nano_4b_pretrain_8gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_pretrain_8gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/nemotron-3-nano-4b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 dist.distributed_timeout_minutes=30 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --load_dir work/model-verification/nemotron-3-nano-4b/pretrain-reference-checkpoints checkpoint.ckpt_step=50 --save_dir work/model-verification/nemotron-3-nano-4b/pretrain-resumed-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-nano-4b-checkpoint-resume-h100", "expected_result": "The continuation restores optimizer, scheduler, data-order, and RNG state directly from iter_0000050 and executes exactly steps 51 through 100 into a distinct output root with the five recorded metrics, finite loss, and no skipped or NaN iterations. Step 51 matches the uninterrupted reference exactly at 6.173212. At step 100, resumed loss 5.512038 differs from reference loss 5.511951 by 0.000087 (0.00158%), inside the declared 1% tolerance. Its complete eight-shard iter_0000100 checkpoint contains metadata, optimizer/RNG train state, run config, and tokenizer, and reloads at step 100 without an extra optimizer step.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 5.512038, "initial_loss": 6.173212, "last_10_steps_model_tflops_per_gpu_avg": 400.11, "last_10_steps_step_time_ms_avg": 26943.07, "last_10_steps_tokens_per_second_per_gpu_avg": 19459.104 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "hf_revision": "dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f", "min_transformers_version": "5.8.1", "slug": "nemotron-3-nano-4b", "source_card": "examples/model_verification_cards/nemotron-3-nano-4b/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Nemotron 3 Nano 4B support verification covers conversion, inference, and training.\n", "title": "nemotron_3_nano_4b" }, { "architecture": "NemotronHForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.06", "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "entries": [ { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 --hf-revision d51eab0d1f979ebc26b546e634a04f450d99158e --megatron-path work/model-verification/nemotron-3-super-120b-a12b/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-hf-to-megatron-cpu", "expected_result": "The pinned CPU import must exit successfully, create iter_0000000, and preserve the complete source checkpoint. A subsequent CPU export from that CPU-imported checkpoint must audit every supported tensor for exact keys, shapes, dtypes, and values; the currently verified CPU export uses the independently verified GPU-imported checkpoint instead.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "75f87e5894d38d01683dfcef8ce6092bf002783e", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 --hf-revision d51eab0d1f979ebc26b546e634a04f450d99158e --megatron-path work/model-verification/nemotron-3-super-120b-a12b/imported-megatron --torch-dtype bfloat16 --tp 1 --pp 1 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-hf-to-megatron-gpu", "expected_result": "The pinned eight-GPU BF16 import exits successfully and creates a reloadable TP1/PP1/EP8/ETP1 iter_0000000 checkpoint. Its paired export exactly reproduces all 42,683 source tensors and 123,611,033,088 elements, including the 41 FP32 router correction-bias tensors.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 --hf-revision d51eab0d1f979ebc26b546e634a04f450d99158e --megatron-path work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --hf-path work/model-verification/nemotron-3-super-120b-a12b/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-megatron-to-hf-cpu", "expected_result": "The single-process CPU export exits successfully and writes 247,222,108,160 bytes across 42,683 tensors and 123,611,033,088 values. Every exported key, shape, dtype, and value is bitwise identical to the pinned Hugging Face BF16 source, with no dtype conversions, and Transformers strictly reloads all 763 state tensors natively as NemotronHForCausalLM.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "75f87e5894d38d01683dfcef8ce6092bf002783e", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 --hf-revision d51eab0d1f979ebc26b546e634a04f450d99158e --megatron-path work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --hf-path work/model-verification/nemotron-3-super-120b-a12b/hf-export --torch-dtype bfloat16 --distributed-save --tp 1 --pp 1 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-megatron-to-hf-gpu", "expected_result": "Strict distributed export exits successfully and writes 247,222,108,160 bytes across all 42,683 tensors. Keys, shapes, dtypes, values, and critical configuration match the pinned source with zero mismatches, including 41 FP32 router correction-bias tensors, and Transformers reloads the result natively as NemotronHForCausalLM with 88 layers.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-manual-forward-pass", "expected_result": "Verification requires a pinned-revision BF16 Hugging Face and Megatron one-step comparison with matching next-token predictions and cosine similarity of at least 0.99. Maximum and mean absolute logit differences must be reported as diagnostics; the helper must first support sharding the approximately 231-GiB Hugging Face checkpoint across H100 ranks.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "75f87e5894d38d01683dfcef8ce6092bf002783e", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "pipeline_parallel": "1", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/hf_to_megatron_generate_text.py --hf_model_path nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 --hf-revision d51eab0d1f979ebc26b546e634a04f450d99158e --megatron_model_path work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --tp 1 --pp 1 --ep 8 --etp 1 --prompt \"The capital of France is\" --max_new_tokens 32" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-inference", "expected_result": "Two independent runs in BF16 reload the imported TP1/PP1/EP8/ETP1\ncheckpoint, exit successfully after exactly 32 new tokens, and produce\nbyte-identical output with this literal completion:\n' Paris.\",\n \"The Eiffel Tower is in Paris.\",\n \"Berlin is the capital of Germany.\",\n \"The Brandenburg Gate is in Berlin.\",\n '\nThe token IDs are 6993, 32876, 1369, 1429, 1784, 1436, 4940, 1299,\n35549, 1395, 1294, 6993, 32876, 1369, 1429, 75613, 1395, 1278, 8961,\n1307, 10787, 32876, 1369, 1429, 1784, 59071, 42969, 1395, 1294, 10874,\n32876, and 1369.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "ffeb5a7ae817a5a6ccfa3850d7dee937faad7f0b", "command_topologies": [ { "gpus_per_node": "8", "nodes": "8", "recipe": "nemotron_3_super_pretrain_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 8 --gpus-per-node 8 --recipe nemotron_3_super_pretrain_config --mode pretrain --dataset megatron-indexed --max_steps 100 'dataset.blend=[[\"work/data/rp2/head_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3-super-120b-a12b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2-tokenizer/tokenizer.model rng.seed=1234 dataset.random_seed=1234 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-pretrain-reference --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3-super-120b-a12b/h100-pretrain-config/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-super-120b-a12b-pretrain-h100", "expected_result": "On exactly 64 H100s, the BF16 support run completes 100 bounded RP2 optimizer steps at TP1/PP2/CP1/EP32/ETP1, GBS/MBS 1280/1, natural routing, HybridEP flex dispatch with 1.10x fixed expert capacity, eager execution, selective recompute of layernorm, moe_act, moe, and core_attn, no optimizer-state CPU offload, and ordered gradient reduction. Loss remains finite from 12.19922 to 5.706429 with zero skipped or NaN iterations. The final ten steps average 29368.690 ms and 236.330 TFLOP/s/GPU. Peak device memory is 58.079 GiB allocated and 70.336 GiB reserved. The resolved configuration persists, and complete iter_0000050 and iter_0000100 checkpoints each contain 64 distributed rank shards plus checkpoint metadata and training state.\n", "hardware": "H100", "last_verified": "2026-08-03", "metrics": { "final_loss": 5.706429, "initial_loss": 12.19922, "last_10_steps_model_tflops_per_gpu_avg": 236.33, "last_10_steps_step_time_ms_avg": 29368.69, "last_10_steps_tokens_per_second_per_gpu_avg": 2789.365 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "2", "nodes": "8", "recipe": "nemotron_3_super_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 8 --gpus-per-node 2 --recipe nemotron_3_super_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --max_steps 100 --seq_length 4096 model.cuda_graph_impl=none --lr 4.5e-4 --min_lr 4.5e-6 --warmup_iters 10 'dataset.blend=[[\"work/data/rp2/head_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3-super-120b-a12b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2-tokenizer/tokenizer.model rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.4 optimizer.overlap_cpu_optimizer_d2h_h2d=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-pretrain-reference --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-pretrain-gb200", "expected_result": "On 16 GB200s, the BF16 support run must complete exactly 100 bounded RP2 optimizer steps at TP8/PP1/CP1/EP16/ETP1 and the recipe's cohort GBS/MBS 1024/1. Full activation recompute, 40% optimizer CPU offload, and eager execution keep the model and optimizer checkpoint within memory. Loss must stay finite with no skipped or NaN iteration, and complete iter_0000050 and iter_0000100 checkpoints must save. The recipe moved from GBS 16 to the cohort GBS 1024, so the earlier metrics no longer describe this command and it needs a rerun.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.GB200", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_super_sft_16gpu_h100_bf16_config", "sequence_length": "8192", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"2d59de1cbd51c0adf384eb906b766d1aee0e0517\"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.75 optimizer.overlap_cpu_optimizer_d2h_h2d=true checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-sft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "nemotron-3-super-120b-a12b-sft-h100", "expected_result": "The immutable-revision Tulu3 run must complete exactly 100 full-SFT steps on 16 H100s at sequence length 8192, TP8/EP16, the recipe's cohort GBS/MBS 32/1, and pad-8 offline packing. Loss must stay finite with no skipped or NaN iteration, and the step-100 checkpoint must save completely for export. The recipe moved from GBS 16 to the cohort GBS 32, so the earlier metrics no longer describe this command and it needs a rerun.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "4", "nodes": "4", "recipe": "nemotron_3_super_sft_16gpu_h100_bf16_config", "sequence_length": "8192", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"2d59de1cbd51c0adf384eb906b766d1aee0e0517\"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.75 optimizer.overlap_cpu_optimizer_d2h_h2d=true checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-sft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "nemotron-3-super-120b-a12b-sft-gb200", "expected_result": "The immutable-revision Tulu3 run must complete exactly 100 full-SFT steps on 16 GB200s at sequence length 8192, TP8/EP16, the recipe's cohort GBS/MBS 32/1, and pad-8 offline packing. Loss must stay finite with no skipped or NaN iteration, and the step-100 checkpoint must save completely. The recipe moved from GBS 16 to the cohort GBS 32, so the earlier metrics no longer describe this command and it needs a rerun.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.GB200", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "expert_parallel": "16", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "2", "pipeline_parallel": "1", "tensor_parallel": "8", "total_gpus": 16 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 2 --gpus-per-node 8 --hf-model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 --hf-revision d51eab0d1f979ebc26b546e634a04f450d99158e --megatron-path work/model-verification/nemotron-3-super-120b-a12b/h100-sft/iter_0000100 --hf-path work/model-verification/nemotron-3-super-120b-a12b/h100-sft-hf --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 8 --pp 1 --ep 16 --etp 1", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/nemotron-3-super-120b-a12b/h100-sft-hf --prompt \"Name the capital of France and explain its role in one sentence.\" --max-new-tokens 45 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-sft-export-inference-h100", "expected_result": "The H100 step-100 SFT checkpoint must export to indexed BF16 safetensors, reload natively as NemotronHForCausalLM, and complete the pinned deterministic 45-token greedy inference before promotion.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "expert_parallel": "16", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "4", "pipeline_parallel": "1", "tensor_parallel": "8", "total_gpus": 16 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 4 --gpus-per-node 4 --hf-model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 --hf-revision d51eab0d1f979ebc26b546e634a04f450d99158e --megatron-path work/model-verification/nemotron-3-super-120b-a12b/gb200-sft/iter_0000100 --hf-path work/model-verification/nemotron-3-super-120b-a12b/gb200-sft-hf --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 8 --pp 1 --ep 16 --etp 1", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/nemotron-3-super-120b-a12b/gb200-sft-hf --prompt \"Name the capital of France and explain its role in one sentence.\" --max-new-tokens 45 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-sft-export-inference-gb200", "expected_result": "The GB200 step-100 full-SFT checkpoint exports as 50 indexed BF16 safetensor shards and reloads natively as NemotronHForCausalLM with 88 layers. Greedy inference produces exactly 45 new tokens and the literal completion \"The capital of France is Paris, which serves as the country's political, cultural, and economic center. It is home to many important government institutions, such as the French National Assembly and the Senate, as well as numerous museums\".\n", "hardware": "GB200", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB200", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "44f3031a2e77ff455eb128efbaa19322ad6397a6", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_super_sft_16gpu_h100_bf16_32k_config", "sequence_length": "32768", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 10 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"2d59de1cbd51c0adf384eb906b766d1aee0e0517\"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-long dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.attention_backend=auto model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=1.0 optimizer.overlap_cpu_optimizer_d2h_h2d=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-long-sft --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "offline" }, "entry_id": "nemotron-3-super-120b-a12b-sft-long-context-h100", "expected_result": "On 16 H100s, the dedicated TP1/PP8/CP2/EP2 recipe completes exactly ten packed 32K SFT steps with pad-4 alignment, full recompute, and full optimizer CPU offload. Loss is finite from 1.168852 to 1.008992 with no skipped or NaN iterations. The ten-step metric window includes the first-step cold-start cost by policy, the resolved configuration is persisted, and the final model-only checkpoint saves successfully.\n", "hardware": "H100", "last_verified": "2026-07-24", "metrics": { "final_loss": 1.008992, "initial_loss": 1.168852, "last_10_steps_model_tflops_per_gpu_avg": 5.58, "last_10_steps_step_time_ms_avg": 132859.58, "last_10_steps_tokens_per_second_per_gpu_avg": 30.83 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "4", "nodes": "4", "recipe": "nemotron_3_super_sft_16gpu_h100_bf16_32k_config", "sequence_length": "32768", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 10 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-long-sft --save_interval 10 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "offline" }, "entry_id": "nemotron-3-super-120b-a12b-sft-long-context-gb200", "expected_result": "A dedicated GB200 run must complete at least ten packed 32K SFT steps with CP2, finite loss, no skipped or NaN iterations, all five metrics, a persisted resolved configuration, and a reloadable final checkpoint.\n", "hardware": "GB200", "last_verified": "2026-07-24", "metrics": { "final_loss": 6.406034, "initial_loss": 7.022207, "last_10_steps_model_tflops_per_gpu_avg": 15.93, "last_10_steps_step_time_ms_avg": 21544.74, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.GB200", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_super_peft_16gpu_h100_bf16_config", "sequence_length": "8192", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"2d59de1cbd51c0adf384eb906b766d1aee0e0517\"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-peft-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=selective 'model.recompute_modules=[core_attn]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-peft --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "nemotron-3-super-120b-a12b-peft-h100", "expected_result": "H100 training has completed 100 finite LoRA steps and saved an adapter, but this leaf remains unverified until the exported adapter reloads natively over the pinned Hugging Face base and produces a nonzero, finite logits effect.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "4", "nodes": "4", "recipe": "nemotron_3_super_peft_16gpu_h100_bf16_config", "sequence_length": "8192", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"2d59de1cbd51c0adf384eb906b766d1aee0e0517\"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-peft-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=selective 'model.recompute_modules=[core_attn]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-peft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "nemotron-3-super-120b-a12b-peft-gb200", "expected_result": "On 16 GB200s, the immutable-revision Tulu3 run completes exactly 100 pad-8 packed LoRA steps with finite loss from 1.23861 to 0.8878498, no skipped or NaN iterations, all five metrics, and a final adapter checkpoint. The adapter contains only 82,304 LoRA tensors targeting linear_qkv, linear_proj, linear_fc1, linear_fc2, in_proj, and out_proj. Native PEFT reload over the pinned base succeeds; all logits are finite and the adapter changes them by maximum absolute 3.1875 and mean absolute 0.472428.\n", "hardware": "GB200", "last_verified": "2026-07-24", "metrics": { "final_loss": 0.8878498, "initial_loss": 1.23861, "last_10_steps_model_tflops_per_gpu_avg": 54.12, "last_10_steps_step_time_ms_avg": 12699.46, "last_10_steps_tokens_per_second_per_gpu_avg": 645.067 }, "precision": "bf16", "source_pointer": "items.peft.GB200", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "8", "nodes": "8", "recipe": "nemotron_3_super_pretrain_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 8 --gpus-per-node 8 --recipe nemotron_3_super_pretrain_config --mode pretrain --dataset megatron-indexed --max_steps 100 'dataset.blend=[[\"work/data/rp2/head_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3-super-120b-a12b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2-tokenizer/tokenizer.model rng.seed=1234 dataset.random_seed=1234 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false --load_dir work/model-verification/nemotron-3-super-120b-a12b/h100-pretrain-reference checkpoint.ckpt_step=50 --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-pretrain-resumed --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3-super-120b-a12b/h100-pretrain-config/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-checkpoint-resume-h100", "expected_result": "A direct H100 continuation must restore model, optimizer, scheduler, data-order, and RNG state from iter_0000050, execute exactly steps 51-100 into a distinct output root, match both declared loss sentinels, record all five metrics, and save a complete iter_0000100 checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "2", "nodes": "8", "recipe": "nemotron_3_super_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 8 --gpus-per-node 2 --recipe nemotron_3_super_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --max_steps 100 --seq_length 4096 model.cuda_graph_impl=none --lr 4.5e-4 --min_lr 4.5e-6 --warmup_iters 10 'dataset.blend=[[\"work/data/rp2/head_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3-super-120b-a12b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2-tokenizer/tokenizer.model rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.4 optimizer.overlap_cpu_optimizer_d2h_h2d=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false --load_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-pretrain-reference checkpoint.ckpt_step=50 --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-pretrain-resumed --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-checkpoint-resume-gb200", "expected_result": "The direct GB200 continuation must restore full model, optimizer, scheduler, data-order, and RNG state from iter_0000050 and execute exactly steps 51-100 into a distinct output root. Each declared loss sentinel must satisfy abs(resumed - reference) <= 1e-6 + 0.01 * abs(reference) against the uninterrupted reference. Its pretrain reference moved to the cohort GBS 1024, so both runs need redoing before this item can be verified.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.GB200", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "ffeb5a7ae817a5a6ccfa3850d7dee937faad7f0b", "command_topologies": [ { "gpus_per_node": "8", "nodes": "8", "recipe": "nemotron_3_super_pretrain_64gpu_h100_bf16_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 8 --gpus-per-node 8 --recipe nemotron_3_super_pretrain_64gpu_h100_bf16_config --max_steps 50 logger.log_throughput=true logger.log_device_memory_used=true logger.save_config_filepath=work/model-verification/nemotron-3-super-120b-a12b/h100-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-pretrain-performance-h100", "expected_result": "On exactly 64 H100s, the canonical BF16 performance recipe must complete 50 mock-data steps at TP1/PP2/CP1/EP32/ETP1, GBS/MBS 1280/1, forced routing, HybridEP flex dispatch with 1.10x fixed expert capacity, eager execution, selective recompute of layernorm, moe_act, moe, and core_attn, no optimizer-state CPU offload, and ordered gradient reduction. All keyed rows have finite loss and throughput with zero skipped or NaN iterations. The final ten steps average 27588.580 ms and 248.610 TFLOP/s/GPU, passing gates of at most 28000 ms and at least 240 TFLOP/s/GPU. Peak device memory is 59.000 GiB allocated and 69.344 GiB reserved, and the resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence.\n", "hardware": "H100", "last_verified": "2026-08-03", "metrics": { "final_loss": 0.006422824, "initial_loss": 12.19134, "last_10_steps_model_tflops_per_gpu_avg": 248.61, "last_10_steps_step_time_ms_avg": 27588.58, "last_10_steps_tokens_per_second_per_gpu_avg": 2969.345 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.H100", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "ae2c24890ecd1cf1d535b9e4aa473af1bc94ab71", "command_topologies": [ { "gpus_per_node": "4", "nodes": "16", "recipe": "nemotron_3_super_pretrain_64gpu_gb200_bf16_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 16 --gpus-per-node 4 --recipe nemotron_3_super_pretrain_64gpu_gb200_bf16_config --max_steps 50 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-pretrain-performance-gb200", "expected_result": "On exactly 64 GB200s, the canonical BF16 performance recipe completes 50 mock-data steps at TP2/PP1/CP1/EP64, GBS/MBS 512/1, HybridEP flex dispatch, and Transformer Engine CUDA graph scopes attn, mamba, moe_router, and moe_preprocess. All 50 keyed rows have finite loss and throughput with zero skipped or NaN iterations. The final ten steps average 9926.350 ms and 559.390 TFLOP/s/GPU, passing gates of at most 11000 ms and at least 500 TFLOP/s/GPU, and the resolved configuration is persisted. Mock data and forced routing make the loss a finiteness check rather than convergence evidence. These numbers are GB200-only evidence.\n", "hardware": "GB200", "last_verified": "2026-07-24", "metrics": { "final_loss": 0.01120061, "initial_loss": 12.17778, "last_10_steps_model_tflops_per_gpu_avg": 559.39, "last_10_steps_step_time_ms_avg": 9926.35, "last_10_steps_tokens_per_second_per_gpu_avg": 3301.113 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "0480586879f2513958fd8634fc529693ae13e536", "command_topologies": [ { "gpus_per_node": "4", "nodes": "16", "recipe": "nemotron_3_super_pretrain_64gpu_gb300_nvfp4_config", "sequence_length": "8192", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 16 --gpus-per-node 4 --recipe nemotron_3_super_pretrain_64gpu_gb300_nvfp4_config --mode pretrain --max_steps 50 --seq_length 8192 logger.save_config_filepath=work/model-verification/nemotron-3-super-120b-a12b/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-super-120b-a12b-pretrain-performance-gb300", "expected_result": "On exactly 64 GB300s, the canonical NVFP4 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 512/1, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.17656 to 0.01398012; the final ten steps average 6357.940 ms, 873.330 TFLOP/s/GPU, and 10307.741 tokens/s/GPU. The resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence.\n", "hardware": "GB300", "last_verified": "2026-08-18", "metrics": { "final_loss": 0.01398012, "initial_loss": 12.17656, "last_10_steps_model_tflops_per_gpu_avg": 873.33, "last_10_steps_step_time_ms_avg": 6357.94, "last_10_steps_tokens_per_second_per_gpu_avg": 10307.741 }, "precision": "nvfp4", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "hf_revision": "d51eab0d1f979ebc26b546e634a04f450d99158e", "min_transformers_version": "5.8.1", "slug": "nemotron-3-super-120b-a12b", "source_card": "examples/model_verification_cards/nemotron-3-super-120b-a12b/card.yaml", "summary": "Performance scope: pretrain_performance.H100 and pretrain_performance.GB200 and pretrain_performance.GB300 track tuned canonical 64-GPU performance recipes, with NVFP4 on GB300. Timing and throughput from functional H100 and GB200 training are support-verification sanity checks rather than optimized performance results. Hardware evidence is scoped strictly by accelerator. Nemotron 3 Super 120B-A12B verification covers GPU import, CPU and GPU export, inference, bounded real-data training, checkpoint resume, and canonical H100, GB200, and GB300 benchmarks. CPU import remains unverified.\n", "title": "nemotron_3_super_120b_a12b" }, { "architecture": "NemotronHForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "entries": [ { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 --hf-revision 624ba927cfbef0427354998700de3d51173c8c04 --megatron-path work/model-verification/nemotron-3-ultra-550b-a55b/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-hf-to-megatron-cpu", "expected_result": "A machine with enough host memory must import the complete pinned BF16 checkpoint, persist iter_0000000, and reload it without missing, unexpected, shape-mismatched, dtype-mismatched, or value-mismatched tensors.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "782c48c3bf3aa1066a1564f1206b3866495e9439", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "3", "pipeline_parallel": "3", "tensor_parallel": "1", "total_gpus": 24 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 3 --gpus-per-node 8 --hf-model nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 --hf-revision 624ba927cfbef0427354998700de3d51173c8c04 --megatron-path work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron --torch-dtype bfloat16 --tp 1 --pp 3 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-hf-to-megatron-gpu", "expected_result": "The command exited successfully and persisted a reloadable iter_0000000 at TP1/PP3/EP8/ETP1. Exact distributed export back to the immutable source revision accounted for all 51,023 Nemotron-H, LatentMoE, Mamba, attention, and MTP tensors and 560,524,603,904 parameters with matching names, shapes, dtypes, and values.\n", "hardware": null, "last_verified": "2026-07-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 --hf-revision 624ba927cfbef0427354998700de3d51173c8c04 --megatron-path work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --hf-path work/model-verification/nemotron-3-ultra-550b-a55b/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-megatron-to-hf-cpu", "expected_result": "The single-process CPU export exits successfully and writes all 51,023 tensors, 560,524,603,904 values, and 1,121,049,257,984 tensor-payload bytes. Keys, shapes, dtypes, and values are bitwise identical to the pinned source with no dtype conversions, preserving its BF16 model weights and FP32 expert-score correction biases. Transformers strictly reloads the output natively as NemotronHForCausalLM with all 927 state tensors and no loading discrepancies.\n", "hardware": null, "last_verified": "2026-08-26", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "782c48c3bf3aa1066a1564f1206b3866495e9439", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "3", "pipeline_parallel": "3", "tensor_parallel": "1", "total_gpus": 24 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 3 --gpus-per-node 8 --hf-model nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 --hf-revision 624ba927cfbef0427354998700de3d51173c8c04 --megatron-path work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --hf-path work/model-verification/nemotron-3-ultra-550b-a55b/hf-export --torch-dtype bfloat16 --distributed-save --tp 1 --pp 3 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-megatron-to-hf-gpu", "expected_result": "The command exited successfully and wrote 225 source-native Hugging Face shards. All 51,023 tensors and 560,524,603,904 parameters matched the pinned source exactly in name, shape, dtype, and value, preserving BF16 model weights and FP32 expert-score correction biases. Semantic configuration, generation configuration, the 131,072-token vocabulary, special tokens, chat template, and representative encodings also matched, and the export reloaded natively with NemotronHConfig and architecture NemotronHForCausalLM.\n", "hardware": null, "last_verified": "2026-07-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-manual-forward-pass", "expected_result": "Historical evidence selected token ID 1256 in both models, decoded as two spaces, with cosine similarity 0.999861, maximum absolute logit difference 0.265625, and mean absolute difference 0.062669. Ultra cannot co-reside as one rank-local Hugging Face model and a distributed Megatron model, and compare.py is intentionally unchanged. A future portable orchestration path must produce equivalent evidence before promotion.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "3", "pipeline_parallel": "3", "tensor_parallel": "1", "total_gpus": 24 } ], "commands": [ "./scripts/inference/infer.sh --nodes 3 --gpus-per-node 8 --task legacy-full-prefix-generation --hf_model_path nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 --hf-revision 624ba927cfbef0427354998700de3d51173c8c04 --megatron_model_path work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --tp 1 --pp 3 --ep 8 --etp 1 --prompt \"Explain why the sky appears blue in one sentence.\" --max_new_tokens 32 --apply-chat-template --thinking-mode disabled --legacy-full-prefix" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-inference", "expected_result": "The public launcher exited successfully after reloading the pinned BF16 checkpoint at TP1/PP3/EP8/ETP1. Against the 32-token maximum, greedy decoding produced exactly 32 tokens over generation steps 0-31 without an earlier EOS and emitted the exact JSON-escaped literal completion \"The user wants a one-sentence explanation for why the sky appears blue.\\u003c/think\\u003eThe sky appears blue because Earth's atmosphere scatters shorter (blue)\".\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-pretrain-h100", "expected_result": "H100 functional pretraining remains unverified until a currently exported public recipe completes a bounded run with all applicable numerical and artifact checks.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null, "peak_allocated_memory_gib": null, "peak_reserved_memory_gib": null }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-ultra-550b-a55b-pretrain-gb200", "expected_result": "GB200 functional pretraining remains unverified until a currently exported public recipe completes a bounded run with all applicable numerical and artifact checks.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.GB200", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [ { "gpus_per_node": "8", "nodes": "24", "recipe": "nemotron_3_ultra_sft_192gpu_h100_bf16_openmathinstruct2_packed_config", "total_gpus": 192 } ], "commands": [ "./scripts/training/train.sh --nodes 24 --gpus-per-node 8 --recipe nemotron_3_ultra_sft_192gpu_h100_bf16_openmathinstruct2_packed_config --mode sft --wait --pretrained_checkpoint work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 scheduler.lr_decay_iters=100 'dataset.hf_dataset.split=\"train_1M[:10%]\"' 'dataset.hf_dataset.load_kwargs={revision:\"469216e3f46f4dacf476b382e192485ea51a143e\"}' dataset.hf_output_root=work/data/openmathinstruct2-ultra-4096-v1 dataset.seed=1234 dataset.offline_packing_specs.pad_seq_to_mult=2 validation.eval_iters=0 validation.eval_interval=0 --save_dir work/model-verification/nemotron-3-ultra-550b-a55b/sft-checkpoints --save_interval 100 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/sft-resolved.yaml logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-ultra-550b-a55b-sft-h100", "expected_result": "The public 192-H100 packed OpenMathInstruct-2 full-SFT recipe must use the pinned dataset revision and bounded train_1M first-ten-percent selection, complete 100 optimizer steps at sequence length 4096 and GBS/MBS 128/1, and preserve pad_seq_to_mult 2. This published Ultra contract is support verification rather than the 8K/GBS8 qwen3_30b_a3b_convergence_v2 cohort. Loss must remain finite with no skipped or NaN iterations, all five metrics, a persisted post-setup configuration, audited supervised-token counts, and a complete reloadable final checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-sft-gb200", "expected_result": "No GB200-specific Ultra full-SFT configuration is verified by this PR. The H100 library recipe must be adapted and validated independently before GB200 SFT support is claimed.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.GB200", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-sft-export-inference-h100", "expected_result": "After full SFT is verified, its final checkpoint must export to a complete BF16 Hugging Face directory, reload natively as NemotronHForCausalLM across enough accelerators, and complete one deterministic greedy generation whose literal output is recorded.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-sft-export-inference-gb200", "expected_result": "GB200 SFT export and inference remain blocked on a verified GB200 full-SFT checkpoint.\n", "hardware": "GB200", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB200", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-sft-long-context-h100", "expected_result": "Megatron Bridge does not publish a dedicated Ultra packed long-context recipe. The only published Ultra full-SFT recipe fixes sequence length 4096 and context parallelism one, so it cannot provide the required joint sequence-packing and CP-greater-than-one evidence and is not relabeled as long-context support.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.sft_long_context.H100", "status": "unsupported", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-sft-long-context-gb200", "expected_result": "Megatron Bridge does not publish a dedicated Ultra packed long-context recipe for GB200.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": null, "source_pointer": "items.sft_long_context.GB200", "status": "unsupported", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [ { "gpus_per_node": "8", "nodes": "4", "recipe": "nemotron_3_ultra_peft_32gpu_h100_bf16_openmathinstruct2_packed_config", "total_gpus": 32 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe nemotron_3_ultra_peft_32gpu_h100_bf16_openmathinstruct2_packed_config --mode lora --wait --pretrained_checkpoint work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 scheduler.lr_decay_iters=100 'dataset.hf_dataset.split=\"train_1M[:10%]\"' 'dataset.hf_dataset.load_kwargs={revision:\"469216e3f46f4dacf476b382e192485ea51a143e\"}' dataset.hf_output_root=work/data/openmathinstruct2-ultra-4096-v1 dataset.seed=1234 dataset.offline_packing_specs.pad_seq_to_mult=2 validation.eval_iters=0 validation.eval_interval=0 --save_dir work/model-verification/nemotron-3-ultra-550b-a55b/peft-checkpoints --save_interval 100 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/peft-resolved.yaml logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-ultra-550b-a55b-peft-h100", "expected_result": "The public 32-H100 packed OpenMathInstruct-2 LoRA recipe must use the same pinned bounded dataset and pad_seq_to_mult 2 as full SFT, match all declared attention, MLP, and Mamba adapter targets, and complete 100 optimizer steps at sequence length 4096 and GBS/MBS 128/1. This published Ultra contract is support verification rather than the 8K/GBS8 qwen3_30b_a3b_convergence_v2 cohort. Loss must remain finite with no skipped or NaN iterations, all five metrics, audited supervised-token counts, a persisted post-setup configuration, and a reloadable adapter checkpoint.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-peft-gb200", "expected_result": "No GB200-specific Ultra PEFT configuration is verified by this PR.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.GB200", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-checkpoint-resume-h100", "expected_result": "Ultra checkpoint resume remains unverified because one full checkpoint exceeds the available verification quota. A future run must restore full optimizer, scheduler, data-order, and RNG state from step 50, continue directly at step 51, and satisfy the declared loss tolerance.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "d352aceda8ed4136f1db787bcf449c1b210a2438", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-checkpoint-resume-gb200", "expected_result": "GB200 checkpoint resume remains unverified until a quota-safe reference and continuation pair completes with matching state and losses.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.GB200", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "0480586879f2513958fd8634fc529693ae13e536", "command_topologies": [ { "gpus_per_node": "4", "nodes": "64", "recipe": "nemotron_3_ultra_pretrain_256gpu_gb200_fp8mx_config", "sequence_length": "8192", "total_gpus": 256 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 64 --gpus-per-node 4 --recipe nemotron_3_ultra_pretrain_256gpu_gb200_fp8mx_config --mode pretrain --max_steps 50 --seq_length 8192 logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/gb200-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-pretrain-performance-gb200", "expected_result": "On exactly 256 GB200s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP2/PP1/CP1/EP64/ETP1, GBS/MBS 256/1, and sequence length 8192 using Megatron-FSDP HSDP. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.1668 to 6.738161; the final ten steps average 3213.970 ms, 917.750 TFLOP/s/GPU, and 2548.873 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB200", "last_verified": "2026-08-19", "metrics": { "final_loss": 6.738161, "initial_loss": 12.1668, "last_10_steps_model_tflops_per_gpu_avg": 917.75, "last_10_steps_step_time_ms_avg": 3213.97, "last_10_steps_tokens_per_second_per_gpu_avg": 2548.873 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "fc704f7809f84461c6e2cbf476acb28c4a5654c0", "command_topologies": [ { "gpus_per_node": "4", "nodes": "64", "recipe": "nemotron_3_ultra_pretrain_256gpu_gb300_fp8mx_config", "sequence_length": "8192", "total_gpus": 256 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 64 --gpus-per-node 4 --recipe nemotron_3_ultra_pretrain_256gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 8192 logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-ultra-550b-a55b-pretrain-performance-gb300", "expected_result": "On exactly 256 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 256/1, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.20118 to 7.019335; the final ten steps average 2282.400 ms, 1306.850 TFLOP/s/GPU, and 3589.204 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB300", "last_verified": "2026-07-25", "metrics": { "final_loss": 7.019335, "initial_loss": 12.20118, "last_10_steps_model_tflops_per_gpu_avg": 1306.85, "last_10_steps_step_time_ms_avg": 2282.4, "last_10_steps_tokens_per_second_per_gpu_avg": 3589.204 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "hf_revision": "624ba927cfbef0427354998700de3d51173c8c04", "min_transformers_version": "5.8.0", "slug": "nemotron-3-ultra-550b-a55b", "source_card": "examples/model_verification_cards/nemotron-3-ultra-550b-a55b/card.yaml", "summary": "Nemotron 3 Ultra 550B-A55B has public packed full-SFT and packed PEFT recipes. Its large-scale BF16 library pretraining workflows remain unverified because the previously recorded 256-GPU recipe names are not exported on current main. The separately scoped pretrain_performance.GB200 and pretrain_performance.GB300 are verified with their canonical 256-GPU MXFP8 recipes. Its pinned 24-GPU BF16 Hugging Face-to-Megatron-to-Hugging Face round trip, single-process CPU export, and checkpoint-backed deterministic inference are verified. CPU import, portable manual-forward, remaining functional training, and checkpoint resume still require completed verification runs before those support claims are promoted.\n", "title": "nemotron_3_ultra_550b_a55b" }, { "architecture": "NemotronHForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "entries": [ { "bridge_commit": "7edd7c7f68bcdd2b30cbaff426ece2ee0da9deb5", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron-path work/model-verification/nemotron-3.5-lightning/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-hf-to-megatron-cpu", "expected_result": "The pinned checkpoint imports on CPU, creates a reloadable iter_0000000, and preserves every MTP tensor for the paired CPU export. After export, all 6,513 tensors and 32,913,266,240 parameters match the pinned source exactly in name, shape, dtype, and value.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --tp 1 --pp 1 --ep 8 --etp 1 --hf-model nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron-path work/model-verification/nemotron-3.5-lightning/imported-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-hf-to-megatron-gpu", "expected_result": "The command exits successfully and creates a reloadable iter_0000000. After the paired GPU export, all 6,513 tensors and 32,913,266,240 parameters match the pinned source exactly in name, shape, dtype, and value, including the MTP layers.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "7edd7c7f68bcdd2b30cbaff426ece2ee0da9deb5", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron-path work/model-verification/nemotron-3.5-lightning/cpu-megatron/iter_0000000 --hf-path work/model-verification/nemotron-3.5-lightning/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-megatron-to-hf-cpu", "expected_result": "Strict CPU export writes a native NemotronHForCausalLM checkpoint that reloads with Transformers. All 6,513 tensors and 32,913,266,240 parameters match the pinned source bitwise with maximum difference zero.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --tp 1 --pp 1 --ep 8 --etp 1 --hf-model nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron-path work/model-verification/nemotron-3.5-lightning/imported-megatron/iter_0000000 --hf-path work/model-verification/nemotron-3.5-lightning/hf-export --torch-dtype bfloat16 --distributed-save" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-megatron-to-hf-gpu", "expected_result": "Strict export exits successfully; all 6,513 tensors match the pinned source bitwise with original dtypes preserved, maximum difference zero, and Transformers reloads the output natively as NemotronHForCausalLM.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "pipeline_parallel": "1", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron_model_path work/model-verification/nemotron-3.5-lightning/imported-megatron/iter_0000000 --tp 1 --pp 1 --ep 8 --etp 1 --prompt \"The capital of France is the city of\"" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-manual-forward-pass", "expected_result": "The pinned-revision one-step comparison exits successfully. Hugging Face and Megatron next-token predictions match at token ID 6993 (\" Paris\"); cosine similarity is 0.998581, above the 0.99 gate. Maximum and mean absolute logit differences are 0.796875 and 0.116128, respectively.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "pipeline_parallel": "1", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/hf_to_megatron_generate_text.py --hf_model_path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron_model_path work/model-verification/nemotron-3.5-lightning/imported-megatron/iter_0000000 --tp 1 --pp 1 --ep 8 --etp 1 --prompt \"The capital of France is\" --max_new_tokens 2" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-inference", "expected_result": "Two independent runs produce byte-identical output with this exact 2-token completion, including its leading space: \" Paris.\".\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "33ede31740aa44183a253a62a4bcf968f176e161", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_config", "sequence_length": "8192", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe nemotron_3_5_lightning_pretrain_config --mode pretrain --dataset megatron-indexed --seq_length 8192 --max_steps 20 --lr 3e-4 --min_lr 3e-5 --warmup_iters 8 'dataset.blend=[[\"work/data/rp2/perp_middle/middle_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3.5-lightning-h100/rp2 rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null --save_dir work/model-verification/nemotron-3.5-lightning-h100/pretrain-reference --save_interval 10 checkpoint.async_save=false logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "cuda_graph": { "implementation": "transformer_engine", "scopes": [ "mamba" ] }, "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-5-lightning-pretrain-h100", "expected_result": "On two complete eight-H100 nodes, the real-data RP2 CP2/p2p run completes exactly 20 sequence-length-8,192, GBS/MBS 512/1 optimizer steps with natural routing and finite LM/MTP-1/MTP-2 losses. LM loss is 12.18927 -> 7.239111, steps 11-20 average 36,869.270 ms / 200.180 TFLOP/s/GPU, no iteration is skipped or NaN, and complete 20-file, 430-GiB checkpoints are written at steps 10 and 20.\n", "hardware": "H100", "last_verified": "2026-07-28", "metrics": { "final_loss": 7.239111, "initial_loss": 12.18927, "last_10_steps_model_tflops_per_gpu_avg": 200.18, "last_10_steps_step_time_ms_avg": 36869.27, "last_10_steps_tokens_per_second_per_gpu_avg": 7110.095, "peak_allocated_memory_gib": 68.547, "peak_reserved_memory_gib": 73.893 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "edca4a7d868cd1214bf5292f28a515139470dd99", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_8k_config", "sequence_length": "8192", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --additional-slurm-params segment=2 --recipe nemotron_3_5_lightning_pretrain_8k_config --mode pretrain --dataset megatron-indexed --seq_length 8192 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 8 'dataset.blend=[[\"work/data/rp2/perp_middle/middle_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3.5-lightning-gb200/rp2 rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/pretrain-reference logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-5-lightning-pretrain-gb200", "expected_result": "On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192, GBS/MBS 512/2 real-data RP2 convergence-verification run completes exactly 100 finite-loss optimizer steps with natural routing, Transformer Engine fused cross entropy, CUDA graphs disabled, finite LM/MTP-1/MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.171970 -> 5.798433; real-data steps 91-100 average 21,483.630 ms / 686.920 TFLOP/s/GPU with 158.220 GiB peak allocated and 162.290 GiB peak reserved memory. Complete 15-file, 430-GiB checkpoints with distributed metadata are written at steps 50 and 100.\n", "hardware": "GB200", "last_verified": "2026-07-28", "metrics": { "final_loss": 5.798433, "initial_loss": 12.17197, "last_10_steps_model_tflops_per_gpu_avg": 686.92, "last_10_steps_step_time_ms_avg": 21483.63, "last_10_steps_tokens_per_second_per_gpu_avg": 24404.07, "peak_allocated_memory_gib": 158.22, "peak_reserved_memory_gib": 162.29 }, "precision": "bf16", "source_pointer": "items.pretrain.GB200", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "edca4a7d868cd1214bf5292f28a515139470dd99", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_8k_fsdp_config", "sequence_length": "8192", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --additional-slurm-params segment=2 --recipe nemotron_3_5_lightning_pretrain_8k_fsdp_config --mode pretrain --dataset megatron-indexed --seq_length 8192 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 8 'dataset.blend=[[\"work/data/rp2/perp_middle/middle_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3.5-lightning-gb200/rp2 rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/pretrain-fsdp logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": { "megatron_fsdp": "optim_grads_params", "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-5-lightning-pretrain-fsdp-gb200-bf16", "expected_result": "On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192, GBS/MBS 512/2 BF16 real-data RP2 FSDP convergence-verification run completes exactly 100 finite-loss optimizer steps with natural routing, Transformer Engine fused cross entropy, active MTP-1/MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.171970 -> 5.797082; real-data steps 91-100 average 26,665.830 ms / 553.430 TFLOP/s/GPU with 160.760 GiB peak allocated and 166.230 GiB peak reserved memory. Complete 14-file, 369-GiB FSDP checkpoints with distributed metadata are written at steps 50 and 100. Do not use this run to claim convergence neutrality against ordinary BF16 pretraining: 188 of 300 shared LM/MTP loss values exceed the 1% loss gate, with a maximum relative difference of 12.95%, despite the similar final loss.\n", "hardware": "GB200", "last_verified": "2026-07-28", "metrics": { "final_loss": 5.797082, "initial_loss": 12.17197, "last_10_steps_model_tflops_per_gpu_avg": 553.43, "last_10_steps_step_time_ms_avg": 26665.83, "last_10_steps_tokens_per_second_per_gpu_avg": 19661.417, "peak_allocated_memory_gib": 160.76, "peak_reserved_memory_gib": 166.23 }, "precision": "bf16", "source_pointer": "items.pretrain_fsdp.GB200.variants.bf16", "status": "verified", "variant": "bf16", "workflow": "pretrain_fsdp" }, { "bridge_commit": "968ab25dd0e8b76801e757ee9976d9806af9c11a", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_8gpu_gb200_fp8mx_fsdp_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 4 --additional-slurm-params segment=2 --recipe nemotron_3_5_lightning_pretrain_8gpu_gb200_fp8mx_fsdp_config --mode pretrain --max_steps 20" ], "dimensions": {}, "enabled_features": { "megatron_fsdp": "optim_grads_params", "moe_dispatcher": "hybridep" }, "entry_id": "nemotron-3-5-lightning-pretrain-fsdp-gb200-fp8-mx", "expected_result": "On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192 MXFP8 FSDP recipe completes exactly 20 finite-loss optimizer steps with GBS/MBS 384/3, active MTP-1 and MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.19034 -> 3.913218, and steps 11-20 average 13,917.000 ms / 795.390 TFLOP/s/GPU with 169.54 GiB peak allocated and 173.86 GiB peak reserved memory. This run must not be compared with the BF16 real-data pretrain or BF16 FSDP runs: it uses MXFP8, mock data, forced routing, GBS/MBS 384/3, and no CUDA graphs.\n", "hardware": "GB200", "last_verified": "2026-07-24", "metrics": { "final_loss": 3.913218, "initial_loss": 12.19034, "last_10_steps_model_tflops_per_gpu_avg": 795.39, "last_10_steps_step_time_ms_avg": 13917.0, "last_10_steps_tokens_per_second_per_gpu_avg": 28254.365, "peak_allocated_memory_gib": 169.54, "peak_reserved_memory_gib": 173.86 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_fsdp.GB200.variants.fp8_mx", "status": "verified", "variant": "fp8_mx", "workflow": "pretrain_fsdp" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_5_lightning_sft_openmathinstruct2_packed_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe nemotron_3_5_lightning_sft_openmathinstruct2_packed_config --mode sft --max_steps 100 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-h100/imported-megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-lightning-h100/sft-checkpoints" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-5-lightning-sft-h100", "expected_result": "Packed OpenMathInstruct-2 full SFT completes 100 finite-loss steps with active MTP gradients, LM loss 0.4406566 -> 0.2757806, finite MTP-1 loss 0.7371141 -> 0.4133309, and no skipped or NaN iterations. Steps 91-100 average 9,887.800 ms / 79.240 TFLOP/s/GPU, and the complete reloadable full-model checkpoint is written at step 100.\n", "hardware": "H100", "last_verified": "2026-08-03", "metrics": { "final_loss": 0.2757806, "initial_loss": 0.4406566, "last_10_steps_model_tflops_per_gpu_avg": 79.24, "last_10_steps_step_time_ms_avg": 9887.8, "last_10_steps_tokens_per_second_per_gpu_avg": 3313.983 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "edca4a7d868cd1214bf5292f28a515139470dd99", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_sft_openmathinstruct2_packed_tp1_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --additional-slurm-params segment=2 --recipe nemotron_3_5_lightning_sft_openmathinstruct2_packed_tp1_config --mode sft --max_steps 100 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-gb200/imported-megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-lightning-gb200/sft-checkpoints" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-5-lightning-sft-gb200", "expected_result": "On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, packed OpenMathInstruct-2 full SFT completes exactly 100 finite-loss optimizer steps with active MTP gradients. LM loss is 0.4407797 -> 0.2762039, MTP-1 loss is 0.7373838 -> 0.4146959, and MTP-2 loss is 0.8092746 -> 0.4744196. Steps 91-100 average 6,795.830 ms / 256.990 TFLOP/s/GPU with 101.510 GiB peak allocated and 103.960 GiB peak reserved memory. No iteration is skipped or NaN, and the complete 12-file reloadable full-model checkpoint is written at step 100.\n", "hardware": "GB200", "last_verified": "2026-08-04", "metrics": { "final_loss": 0.2762039, "initial_loss": 0.4407797, "last_10_steps_model_tflops_per_gpu_avg": 256.99, "last_10_steps_step_time_ms_avg": 6795.83, "last_10_steps_tokens_per_second_per_gpu_avg": 9643.561, "peak_allocated_memory_gib": 101.51, "peak_reserved_memory_gib": 103.96 }, "precision": "bf16", "source_pointer": "items.sft.GB200", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "2", "pipeline_parallel": "1", "tensor_parallel": "2", "total_gpus": 16 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 2 --gpus-per-node 8 --tp 2 --pp 1 --ep 8 --etp 1 --hf-model nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron-path work/model-verification/nemotron-3.5-lightning-h100/sft-checkpoints/iter_0000100 --hf-path work/model-verification/nemotron-3.5-lightning-h100/sft-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/nemotron-3.5-lightning-h100/sft-hf-export --prompt \"Solve briefly: If 3x + 5 = 20, what is x?\" --max-new-tokens 45 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-sft-export-inference-h100", "expected_result": "The step-100 SFT checkpoint exports 6,513 tensors, including all 270 MTP tensors, to native Hugging Face format and reloads as NemotronHForCausalLM. The deterministic verifier performs one greedy generation and produces exactly 45 new tokens. The literal completion is: \"To solve for $x$, follow these steps:\n\n1. Subtract 5 from both sides of the equation:\n\\[ 3x + 5 - 5 = 20 - 5 \\Rightarrow 3x\".", "hardware": "H100", "last_verified": "2026-08-03", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "edca4a7d868cd1214bf5292f28a515139470dd99", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "2", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 2 --gpus-per-node 4 --tp 1 --pp 1 --ep 8 --etp 1 --hf-model nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --hf-revision b3caaabed0263651a17dc1f2d4ce97e794f76c44 --megatron-path work/model-verification/nemotron-3.5-lightning-gb200/sft-checkpoints/iter_0000100 --hf-path work/model-verification/nemotron-3.5-lightning-gb200/sft-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/nemotron-3.5-lightning-gb200/sft-hf-export --prompt \"Solve briefly: If 3x + 5 = 20, what is x?\" --max-new-tokens 45 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-sft-export-inference-gb200", "expected_result": "The step-100 SFT checkpoint exports 6,513 tensors, including all 270 MTP tensors, to native Hugging Face format and reloads as NemotronHForCausalLM. The deterministic verifier performs one greedy generation and produces exactly 45 new tokens. The literal completion is: \"To solve for $x$, follow these steps:\n\n1. Subtract 5 from both sides of the equation:\n\\[ 3x + 5 - 5 = 20 - 5 \\Rightarrow 3x\".", "hardware": "GB200", "last_verified": "2026-08-04", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB200", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "context_parallel": "2", "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_5_lightning_sft_config", "sequence_length": "32768", "tensor_parallel": "2", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe nemotron_3_5_lightning_sft_config --mode sft --dataset openmathinstruct2 --max_steps 100 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --context_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-h100/imported-megatron/iter_0000000 'dataset.hf_dataset.load_kwargs={revision:\"469216e3f46f4dacf476b382e192485ea51a143e\"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.sequence_parallel=true model.cp_comm_type=p2p model.cross_entropy_loss_fusion=true model.calculate_per_token_loss=true model.recompute_granularity=full model.recompute_modules=null model.recompute_method=uniform model.recompute_num_layers=1 model.moe_router_force_load_balancing=false ddp.average_in_collective=false ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-h100/long-sft-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-5-lightning-sft-long-context-h100", "expected_result": "Packed OpenMathInstruct-2 CP2 SFT completes exactly 100 finite-loss 32K steps with LM loss 0.4268321 -> 0.2650192 and MTP-1 loss 0.7322379 -> 0.4439273. All 100 steps have finite LM/MTP losses and no skipped or NaN iterations. Steps 91-100 average 40,863.010 ms / 188.950 TFLOP/s/GPU. Complete step-50 and step-100 checkpoints each contain 20 files, distributed metadata, saved run configuration and training state, with latest marker 100.\n", "hardware": "H100", "last_verified": "2026-08-03", "metrics": { "final_loss": 0.2650192, "initial_loss": 0.4268321, "last_10_steps_model_tflops_per_gpu_avg": 188.95, "last_10_steps_step_time_ms_avg": 40863.01, "last_10_steps_tokens_per_second_per_gpu_avg": 6415.191 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "context_parallel": "2", "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_sft_config", "sequence_length": "32768", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe nemotron_3_5_lightning_sft_config --mode sft --dataset openmathinstruct2 --max_steps 100 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 --context_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-gb200/imported-megatron/iter_0000000 model.moe_flex_dispatcher_backend=hybridep 'dataset.hf_dataset.load_kwargs={revision:\"469216e3f46f4dacf476b382e192485ea51a143e\"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.calculate_per_token_loss=true model.recompute_granularity=selective 'model.recompute_modules=[moe,layernorm,core_attn,mlp]' model.recompute_method=null model.recompute_num_layers=null model.moe_router_force_load_balancing=false ddp.average_in_collective=false ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/long-sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-5-lightning-sft-long-context-gb200", "expected_result": "Packed OpenMathInstruct-2 CP2 SFT completes exactly 100 finite-loss 32K steps with LM loss 0.4268995 -> 0.2649925 and MTP-1 loss 0.7322624 -> 0.4439029. All 100 steps have finite LM/MTP losses and no skipped or NaN iterations. Steps 91-100 average 129,692.530 ms / 119.030 TFLOP/s/GPU. The complete step-100 checkpoint has distributed metadata, saved run configuration and training state, 12 files, and a latest marker of 100.\n", "hardware": "GB200", "last_verified": "2026-08-04", "metrics": { "final_loss": 0.2649925, "initial_loss": 0.4268995, "last_10_steps_model_tflops_per_gpu_avg": 119.03, "last_10_steps_step_time_ms_avg": 129692.53, "last_10_steps_tokens_per_second_per_gpu_avg": 4042.546 }, "precision": "bf16", "source_pointer": "items.sft_long_context.GB200", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "recipe": "nemotron_3_5_lightning_peft_config", "sequence_length": "4096", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe nemotron_3_5_lightning_peft_config --mode lora --dataset openmathinstruct2 --max_steps 100 --seq_length 4096 --lr 1e-4 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-h100/imported-megatron/iter_0000000 'dataset.hf_dataset.load_kwargs={revision:\"469216e3f46f4dacf476b382e192485ea51a143e\"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:4096,pad_seq_to_mult:2}' model.sequence_parallel=true ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-h100/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-5-lightning-peft-h100", "expected_result": "Packed rank-32, alpha-32, zero-dropout LoRA completes 100 finite-loss steps with adapters on the MTP layers, LM loss 0.4406696 -> 0.2854756, finite MTP-1 loss 0.7370828 -> 0.4362708, and no skipped or NaN iterations. The complete adapter checkpoint is written at step 100 and reloads over the pinned base checkpoint at step 100.\n", "hardware": "H100", "last_verified": "2026-08-03", "metrics": { "final_loss": 0.2854756, "initial_loss": 0.4406696, "last_10_steps_model_tflops_per_gpu_avg": 92.89, "last_10_steps_step_time_ms_avg": 16900.36, "last_10_steps_tokens_per_second_per_gpu_avg": 3877.787 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "619a2df1da9517b4408ea475b428bf2aba618c8f", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_peft_config", "sequence_length": "4096", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe nemotron_3_5_lightning_peft_config --mode lora --dataset openmathinstruct2 --max_steps 100 --seq_length 4096 --lr 1e-4 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-gb200/imported-megatron/iter_0000000 model.moe_flex_dispatcher_backend=hybridep 'dataset.hf_dataset.load_kwargs={revision:\"469216e3f46f4dacf476b382e192485ea51a143e\"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:4096,pad_seq_to_mult:2}' model.sequence_parallel=true ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "nemotron-3-5-lightning-peft-gb200", "expected_result": "Packed rank-32, alpha-32, zero-dropout LoRA completes 100 finite-loss steps with 48 rank-local adapter attachment records covering MTP attention, expert, and shared-expert layers, LM loss 0.4406088 -> 0.2855029, finite MTP-1 loss 0.7371238 -> 0.4362065, and no skipped or NaN iterations. The complete adapter checkpoint is written at step 100 and reloads over the pinned base checkpoint at step 100.\n", "hardware": "GB200", "last_verified": "2026-08-04", "metrics": { "final_loss": 0.2855029, "initial_loss": 0.4406088, "last_10_steps_model_tflops_per_gpu_avg": 81.78, "last_10_steps_step_time_ms_avg": 19230.4, "last_10_steps_tokens_per_second_per_gpu_avg": 3407.937 }, "precision": "bf16", "source_pointer": "items.peft.GB200", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "33ede31740aa44183a253a62a4bcf968f176e161", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_config", "sequence_length": "8192", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe nemotron_3_5_lightning_pretrain_config --mode pretrain --dataset megatron-indexed --seq_length 8192 --max_steps 20 --lr 3e-4 --min_lr 3e-5 --warmup_iters 8 'dataset.blend=[[\"work/data/rp2/perp_middle/middle_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3.5-lightning-h100/rp2 rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false --load_dir work/model-verification/nemotron-3.5-lightning-h100/pretrain-reference checkpoint.ckpt_step=10 checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-h100/pretrain-resumed --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-checkpoint-resume-h100", "expected_result": "The continuation restores optimizer, scheduler, data-order, and RNG state from step 10, begins at step 11, finishes at step 20 in a distinct output root, and writes a complete step-20 checkpoint. Resumed losses 7.981691 and 7.238846 match uninterrupted losses 7.981684 and 7.239111 at steps 11 and 20 within the declared tolerance. All 30 shared LM/MTP-1/MTP-2 values pass with a maximum relative difference of 0.0851%, with no skipped or NaN iterations, and the 20-file, 430-GiB step-20 checkpoint is complete.\n", "hardware": "H100", "last_verified": "2026-07-28", "metrics": { "final_loss": 7.238846, "initial_loss": 7.981691, "last_10_steps_model_tflops_per_gpu_avg": 176.63, "last_10_steps_step_time_ms_avg": 56671.64, "last_10_steps_tokens_per_second_per_gpu_avg": 4625.665 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "edca4a7d868cd1214bf5292f28a515139470dd99", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_8k_config", "sequence_length": "8192", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --additional-slurm-params segment=2 --recipe nemotron_3_5_lightning_pretrain_8k_config --mode pretrain --dataset megatron-indexed --seq_length 8192 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 8 'dataset.blend=[[\"work/data/rp2/perp_middle/middle_01_text_document\"],null]' dataset.path_to_cache=work/cache/nemotron-3.5-lightning-gb200/rp2 rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false --load_dir work/model-verification/nemotron-3.5-lightning-gb200/pretrain-reference checkpoint.ckpt_step=50 checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/pretrain-resumed logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null dist.distributed_timeout_minutes=120" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-checkpoint-resume-gb200", "expected_result": "The continuation restores optimizer, scheduler, data-order, and RNG state from the matched 8K step-50 reference, begins at step 51, and finishes at step 100 in a distinct output root. Resumed losses 6.166278 and 5.796845 match uninterrupted losses 6.166278 and 5.798433 at steps 51 and 100. All 150 shared LM/MTP-1/MTP-2 values pass with a maximum relative difference of 0.127978%, no iteration is skipped or NaN, and the 15-file, 430-GiB step-100 checkpoint is complete.\n", "hardware": "GB200", "last_verified": "2026-07-28", "metrics": { "final_loss": 5.796845, "initial_loss": 6.166278, "last_10_steps_model_tflops_per_gpu_avg": 663.86, "last_10_steps_step_time_ms_avg": 22233.45, "last_10_steps_tokens_per_second_per_gpu_avg": 23581.046 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.GB200", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "573e088c9c6740082c39744e03dc5b009e730ed4", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_16gpu_h100_bf16_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe nemotron_3_5_lightning_pretrain_16gpu_h100_bf16_config --max_steps 50" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-pretrain-performance-h100", "expected_result": "On 16x H100, the exact mock-data, force-balanced BF16 performance recipe completes exactly 50 steps with finite LM/MTP-1/MTP-2 losses and no skipped or NaN iterations. Transformer Engine CUDA graphs cover attention and Mamba scopes, while expert-FC1 activations are offloaded. Steps 41-50 average 18,467.280 ms and 399.580 model TFLOP/s/GPU; peak rank-0 allocated/reserved memory is 72.066/75.340 GiB. This benchmark-only run stores Adam moments in BF16, so its losses are not convergence-comparable with FP32 optimizer-state runs.\n", "hardware": "H100", "last_verified": "2026-08-03", "metrics": { "final_loss": 0.01217863, "initial_loss": 12.16197, "last_10_steps_model_tflops_per_gpu_avg": 399.58, "last_10_steps_step_time_ms_avg": 18467.28, "last_10_steps_tokens_per_second_per_gpu_avg": 14195.052, "peak_allocated_memory_gib": 72.066, "peak_reserved_memory_gib": 75.34 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.H100", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "9b0c2dff42e6175e91d34cd2a504529f803ebde9", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "nemotron_3_5_lightning_pretrain_8gpu_gb200_bf16_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe nemotron_3_5_lightning_pretrain_8gpu_gb200_bf16_config --max_steps 50" ], "dimensions": {}, "enabled_features": {}, "entry_id": "nemotron-3-5-lightning-pretrain-performance-gb200", "expected_result": "On 8x GB200, the exact mock-data BF16 performance recipe completes exactly 50 steps with finite LM/MTP-1/MTP-2 losses and no skipped or NaN iterations. Transformer Engine CUDA graphs cover attention, Mamba, MoE-router, and MoE-preprocess scopes. Steps 41-50 average 18,236.060 ms and 809.220 model TFLOP/s/GPU; peak rank-0 allocated/reserved memory is 156.850/161.260 GiB.\n", "hardware": "GB200", "last_verified": "2026-07-28", "metrics": { "final_loss": 0.03325584, "initial_loss": 12.1825, "last_10_steps_model_tflops_per_gpu_avg": 809.22, "last_10_steps_step_time_ms_avg": 18236.06, "last_10_steps_tokens_per_second_per_gpu_avg": 28750.07, "peak_allocated_memory_gib": 156.85, "peak_reserved_memory_gib": 161.26 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "hf_revision": "b3caaabed0263651a17dc1f2d4ce97e794f76c44", "min_transformers_version": "5.8.1", "slug": "nemotron-3.5-lightning", "source_card": "examples/model_verification_cards/nemotron-3.5-lightning/card.yaml", "summary": "Performance scope: pretrain_performance.H100 and pretrain_performance.GB200 use tuned canonical performance recipes; timing and throughput metrics from functional training items remain sanity checks rather than optimized performance results. The convergence pretraining recipes retain real tokenization, real data, natural routing, safety checks, and checkpointing while adopting the corresponding hardware performance recipe's execution stack. The H100 performance leaf uses BF16 Adam moments in a benchmark-only workload and is not convergence-comparable with FP32 optimizer-state runs. Nemotron 3.5 Lightning support verification covers conversion, inference, and training on H100 and GB200. Recipe names, work paths, and this verification-card path use the public Lightning name. The evidence used the 26.08 release-candidate software stack; the environment records its corresponding public 26.08 base container.\n", "title": "nemotron_3_5_lightning" }, { "architecture": "GptOssForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.06", "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "entries": [ { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model openai/gpt-oss-120b --hf-revision b5c939de8f754692c1647ca79fbf85e8c1e70f8a --megatron-path work/model-verification/gpt-oss-120b/cpu-megatron --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-hf-to-megatron-cpu", "expected_result": "Future verification must import the pinned MXFP4 Hugging Face checkpoint, dequantize it into a reloadable BF16 Megatron checkpoint at iter_0000000, and audit keys, shapes, dtypes, and values against the recorded source revision.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model openai/gpt-oss-120b --hf-revision b5c939de8f754692c1647ca79fbf85e8c1e70f8a --megatron-path work/model-verification/gpt-oss-120b/imported-megatron --torch-dtype bfloat16 --tp 1 --pp 4 --ep 8 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-hf-to-megatron-gpu", "expected_result": "Imports the pinned MXFP4 checkpoint on 32 H100 GPUs and creates a reloadable TP1/PP4/EP8/ETP1 BF16 Megatron checkpoint at iter_0000000 with 32 DistCP shards, metadata, train-state, tokenizer, and latest-marker artifacts.\n", "hardware": null, "last_verified": "2026-08-10", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model unsloth/gpt-oss-120b-BF16 --hf-revision e7523373bc44b42296b43202e265a1eebf2ee16f --megatron-path work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --hf-path work/model-verification/gpt-oss-120b/cpu-hf-export --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-megatron-to-hf-cpu", "expected_result": "The single-process CPU export exits successfully and writes the complete BF16 Hugging Face checkpoint. Exhaustive comparison against the pinned unsloth/gpt-oss-120b-BF16 reference covers all 615 tensors and 116,829,156,672 values with exact keys, shapes, dtypes, and bitwise-equal values. Transformers strictly reloads all 615 state tensors natively as GptOssForCausalLM with attention_bias=true and no loading discrepancies, and the tokenizer also reloads successfully.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model unsloth/gpt-oss-120b-BF16 --megatron-path work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --hf-path work/model-verification/gpt-oss-120b/hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 1 --pp 4 --ep 8 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-megatron-to-hf-gpu", "expected_result": "Exports the distributed Megatron checkpoint to a reloadable BF16 Hugging Face checkpoint with attention_bias=true. The exported checkpoint has 73 safetensors shards and 615 BF16 tensors totaling 116,829,156,672 parameters, with no missing, unexpected, shape, dtype, or value mismatches against unsloth/gpt-oss-120b-BF16.\n", "hardware": null, "last_verified": "2026-08-10", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/inference/infer.sh --nodes 4 --gpus-per-node 8 --task model-comparison --hf_model_path openai/gpt-oss-120b --hf-revision b5c939de8f754692c1647ca79fbf85e8c1e70f8a --megatron_model_path work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --hf-logits-path work/model-verification/gpt-oss-120b/manual-forward/hf_logits_france.pt --tp 1 --pp 4 --ep 8 --etp 1 --prompt \"The capital of France is the city of\" --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-manual-forward-pass", "expected_result": "Compares pinned OpenAI Hugging Face reference logits and the imported Megatron checkpoint on the prompt \"The capital of France is the city of\". Both predict next token 12650 (' Paris'); token match is true, cosine similarity is 0.999963, maximum absolute logit difference is 0.250000, and mean absolute logit difference is 0.022671.\n", "hardware": null, "last_verified": "2026-08-10", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/inference/infer.sh --nodes 4 --gpus-per-node 8 --task legacy-full-prefix-generation --legacy-full-prefix --hf_model_path unsloth/gpt-oss-120b-BF16 --megatron_model_path work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --prompt \"The capital of France is\" --max_new_tokens 32 --tp 1 --pp 4 --ep 8 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-inference", "expected_result": "Completes one synchronous deterministic greedy generation through the\nnon-cached compatibility path. Generated exactly 32 new tokens with this\nliteral completion \"\n Paris.\n\nGreat! If you have any more questions or need further assistance, feel free to ask!\n\nSure!\n\nIf there's anything else you'd like to\n\"\n", "hardware": null, "last_verified": "2026-08-10", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "gpus_per_node": "8", "nodes": "8", "recipe": "gpt_oss_120b_pretrain_config", "sequence_length": "4096", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --nodes 8 --gpus-per-node 8 --recipe gpt_oss_120b_pretrain_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/gpt-oss-120b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load=null checkpoint.finetune=false checkpoint.save_optim=true checkpoint.save_rng=true --save_dir work/model-verification/gpt-oss-120b/pretrain-reference-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-pretrain-h100", "expected_result": "On 64x H100, the command completes exactly 100 bounded pretraining optimizer steps using the recipe-owned GBS/MBS, natural routing, and BF16 configuration. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 10.91136 to 5.828481, all five metrics are recorded, the post-setup run_config.yaml persists, and complete 64-shard iter_0000050 and iter_0000100 checkpoints are saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers suitable for direct resume.\n", "hardware": "H100", "last_verified": "2026-08-11", "metrics": { "final_loss": 5.828481, "initial_loss": 10.91136, "last_10_steps_model_tflops_per_gpu_avg": 58.42, "last_10_steps_step_time_ms_avg": 16664.93, "last_10_steps_tokens_per_second_per_gpu_avg": 1966.285 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "gpus_per_node": "8", "nodes": "4", "recipe": "gpt_oss_120b_sft_32gpu_h100_bf16_config", "sequence_length": "2048", "tensor_parallel": "2", "total_gpus": 32 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe gpt_oss_120b_sft_32gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"b5c939de8f754692c1647ca79fbf85e8c1e70f8a\"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-sft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 model.tensor_model_parallel_size=2 model.sequence_parallel=true model.recompute_granularity=selective 'model.recompute_modules=[moe_act]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-120b/sft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "gpt-oss-120b-sft-h100", "expected_result": "Completed 100 full-SFT steps from the imported checkpoint on 32x H100 with immutable Tulu3 selection, assistant-token masking, offline packing, TP2/SP, and selective MoE-activation recompute. All 100 optimizer-step rows are present with finite losses, zero skipped iterations, and zero NaN iterations; loss decreases from 6.751403 to 0.7468084. All five metrics are recorded, the checkpoint-local post-setup run_config.yaml persists, and the final full-model checkpoint at iter_0000100 is complete with 32 DistCP shards, metadata, train state, tokenizer artifacts, and latest-checkpoint marker 100.\n", "hardware": "H100", "last_verified": "2026-08-12", "metrics": { "final_loss": 0.7468084, "initial_loss": 6.751403, "last_10_steps_model_tflops_per_gpu_avg": 33.8, "last_10_steps_step_time_ms_avg": 8063.15, "last_10_steps_tokens_per_second_per_gpu_avg": 1015.98 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "2", "total_gpus": 32 }, { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model unsloth/gpt-oss-120b-BF16 --megatron-path work/model-verification/gpt-oss-120b/sft-2k-checkpoints/iter_0000100 --hf-path work/model-verification/gpt-oss-120b/sft-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 2 --pp 4 --ep 8 --etp 1 --trust-remote-code", "./scripts/inference/infer.sh --task hf-inference --nodes 1 --gpus-per-node 1 --hf-model work/model-verification/gpt-oss-120b/sft-hf-export --prompt \"Name the capital of France and explain its role in one sentence.\" --max-new-tokens 45 --chat-template --trust-remote-code --device cpu" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-sft-export-inference-h100", "expected_result": "The verified step-100 SFT checkpoint exports to a strict-reloadable BF16\nHugging Face checkpoint with architecture GptOssForCausalLM and\nattention_bias=true. The exported checkpoint has 73 safetensors shards,\n615 tensors, model.safetensors.index.json total_size 233,658,313,344,\ntokenizer artifacts, generation_config.json, and chat_template.jinja.\nThe deterministic HF verifier reloads the export on CPU through\nTransformers with no missing, unexpected, mismatched, or error loading\nkeys and generates exactly 45 new tokens under the 45-token bound. The\nliteral completion is:\n\"finalThe capital of France is Paris, and it serves as the political, economic, and cultural center of the country, housing important institutions such as the French government, major corporations, and world-renowned museums and landmarks.\"\n", "hardware": "H100", "last_verified": "2026-08-13", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "gpus_per_node": "8", "nodes": "6", "recipe": "gpt_oss_120b_sft_48gpu_h100_bf16_32k_config", "sequence_length": "32768", "total_gpus": 48 } ], "commands": [ "./scripts/training/train.sh --nodes 6 --gpus-per-node 8 --recipe gpt_oss_120b_sft_48gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"b5c939de8f754692c1647ca79fbf85e8c1e70f8a\"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-sft-long-context-32k-cp4-pp6-nofuse-gbs4-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=8 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 4, "sequence_packing": "offline" }, "entry_id": "gpt-oss-120b-sft-long-context-h100", "expected_result": "Completed 20 long-context full-SFT optimizer steps from the imported checkpoint on 48x H100 with 32K sequence length, immutable Tulu3 selection, assistant-token masking, offline packing, TP2/SP, CP4 with a2a context-parallel communication, PP6, EP4, disabled cross-entropy and bias-activation fusions, and selective MoE-activation recompute. All 20 optimizer-step rows are present exactly once with finite losses, zero skipped iterations, and zero NaN iterations. Loss decreases from 6.666666 to 1.238030, all five metrics are recorded, and no checkpoint is saved because this bounded long-context verification run disables checkpoint output.\n", "hardware": "H100", "last_verified": "2026-08-13", "metrics": { "final_loss": 1.23803, "initial_loss": 6.666666, "last_10_steps_model_tflops_per_gpu_avg": 5.07, "last_10_steps_step_time_ms_avg": 24388.78, "last_10_steps_tokens_per_second_per_gpu_avg": 111.964 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "gpt_oss_120b_peft_8gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_120b_peft_8gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"b5c939de8f754692c1647ca79fbf85e8c1e70f8a\"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-peft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-120b/peft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "gpt-oss-120b-peft-h100", "expected_result": "Completed 100 LoRA steps from the imported checkpoint with the base frozen, model-native target modules, immutable Tulu3 selection, assistant-token masking, and offline packing. All 100 optimizer-step rows are present with finite losses, zero skipped iterations, and zero NaN iterations; the checkpoint-local post-setup run_config.yaml persists and latest_checkpointed_iteration.txt points to the reloadable final adapter checkpoint at iter_0000100.\n", "hardware": "H100", "last_verified": "2026-08-11", "metrics": { "final_loss": 0.9883968, "initial_loss": 6.667154, "last_10_steps_model_tflops_per_gpu_avg": 89.85, "last_10_steps_step_time_ms_avg": 11509.1, "last_10_steps_tokens_per_second_per_gpu_avg": 2847.138 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "gpus_per_node": "8", "nodes": "8", "recipe": "gpt_oss_120b_pretrain_config", "sequence_length": "4096", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --nodes 8 --gpus-per-node 8 --recipe gpt_oss_120b_pretrain_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/gpt-oss-120b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.ckpt_step=50 checkpoint.finetune=false --load_dir work/model-verification/gpt-oss-120b/pretrain-reference-checkpoints --save_dir work/model-verification/gpt-oss-120b/pretrain-resume-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-checkpoint-resume-h100", "expected_result": "The command restores optimizer, scheduler, data-order, and RNG state directly from the verified pretrain step-50 checkpoint into a distinct output root and executes exactly steps 51 through 100. All 50 resumed optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. The step-51 loss matches the reference exactly at 6.489011, and the step-100 loss 5.825046 is within the declared one percent relative plus 1e-6 absolute gate against reference loss 5.828481. All five metrics are recorded, the post-setup run_config.yaml persists, and the final step-100 checkpoint is complete with 64 DistCP shards, metadata, train state, tokenizer artifacts, and latest-checkpoint marker 100.\n", "hardware": "H100", "last_verified": "2026-08-11", "metrics": { "final_loss": 5.825046, "initial_loss": 6.489011, "last_10_steps_model_tflops_per_gpu_avg": 58.56, "last_10_steps_step_time_ms_avg": 16625.3, "last_10_steps_tokens_per_second_per_gpu_avg": 1970.972 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "gpus_per_node": "4", "nodes": "16", "recipe": "gpt_oss_120b_pretrain_64gpu_gb200_bf16_config", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --nodes 16 --gpus-per-node 4 --recipe gpt_oss_120b_pretrain_64gpu_gb200_bf16_config --mode pretrain --max_steps 50 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-pretrain-performance-gb200", "expected_result": "On exactly 64 GB200s, the canonical BF16 mock-data performance recipe completes 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 1280/4, forced router balancing, selective layernorm and MoE activation recompute, and expert-parallel communication overlap. All 50 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 12.79104 to 0.03246120. The final ten logged step times average 4995.630 ms, measured model throughput averages 535.230 TFLOP/s/GPU, and derived token-slot throughput averages 16398.332 tokens/s/GPU over the same final-ten window. The resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence.\n", "hardware": "GB200", "last_verified": "2026-08-17", "metrics": { "final_loss": 0.0324612, "initial_loss": 12.79104, "last_10_steps_model_tflops_per_gpu_avg": 535.23, "last_10_steps_step_time_ms_avg": 4995.63, "last_10_steps_tokens_per_second_per_gpu_avg": 16398.332 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "0480586879f2513958fd8634fc529693ae13e536", "command_topologies": [ { "gpus_per_node": "4", "nodes": "16", "recipe": "gpt_oss_120b_pretrain_64gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 64 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 16 --gpus-per-node 4 --recipe gpt_oss_120b_pretrain_64gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/gpt-oss-120b/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-120b-pretrain-performance-gb300", "expected_result": "On exactly 64 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP16/ETP1, GBS/MBS 1280/4, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.78424 to 0.0985439; the final ten steps average 2481.140 ms, 1077.350 TFLOP/s/GPU, and 33017.081 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB300", "last_verified": "2026-08-19", "metrics": { "final_loss": 0.0985439, "initial_loss": 12.78424, "last_10_steps_model_tflops_per_gpu_avg": 1077.35, "last_10_steps_step_time_ms_avg": 2481.14, "last_10_steps_tokens_per_second_per_gpu_avg": 33017.081 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "openai/gpt-oss-120b", "hf_revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a", "min_transformers_version": "5.8.0", "slug": "gpt-oss-120b", "source_card": "examples/model_verification_cards/gpt-oss-120b/card.yaml", "summary": "Performance scope: pretrain_performance.GB200 and pretrain_performance.GB300 use tuned canonical performance recipes; timing and throughput metrics from functional training items remain sanity checks rather than optimized performance results. GPT-OSS 120B H100 GPU HF-to-Megatron import, GPU Megatron-to-HF export, manual HF/Megatron forward-pass correlation, deterministic Megatron inference, bounded H100 pretraining, H100 SFT, H100 PEFT, and H100 checkpoint resume, H100 SFT export/inference, and H100 long-context SFT are verified. GB200 and GB300 pretraining performance are verified. CPU Megatron-to-Hugging-Face export is verified; CPU Hugging-Face-to-Megatron import remains planned.\n", "title": "gpt_oss_120b" }, { "architecture": "GptOssForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.06", "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "entries": [ { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model openai/gpt-oss-20b --hf-revision 6cee5e81ee83917806bbde320786a8fb61efebee --megatron-path work/model-verification/gpt-oss-20b/cpu-megatron --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-hf-to-megatron-cpu", "expected_result": "Future verification must import the pinned MXFP4 Hugging Face checkpoint, dequantize it into a reloadable BF16 Megatron checkpoint at iter_0000000, and audit keys, shapes, dtypes, and values against the recorded source revision.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "expert_parallel": "2", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model openai/gpt-oss-20b --hf-revision 6cee5e81ee83917806bbde320786a8fb61efebee --megatron-path work/model-verification/gpt-oss-20b/imported-megatron --torch-dtype bfloat16 --tp 2 --pp 2 --ep 2 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-hf-to-megatron-gpu", "expected_result": "The command exits successfully, imports the pinned MXFP4 Hugging Face checkpoint on 8 GPUs, and creates iter_0000000 as a distributed TP2/PP2/EP2/ETP1 Megatron checkpoint with eight DistCP shards, metadata, train state, tokenizer artifacts, and latest-checkpoint markers.\n", "hardware": null, "last_verified": "2026-08-09", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "60442bb9adb5435b47db22c6c20aacdf772fbddc", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model unsloth/gpt-oss-20b-BF16 --hf-revision cc89b3e7fd423253264883a80a4fa5abc619649f --megatron-path work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --hf-path work/model-verification/gpt-oss-20b/cpu-hf-export --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-megatron-to-hf-cpu", "expected_result": "The synchronous CPU workflow exports the verified distributed Megatron checkpoint to the pinned public unquantized BF16 Hugging Face layout. Its 411 tensors, shapes, dtypes, and all 20,914,757,184 values match the BF16 reference bitwise at zero tolerance. Native Transformers loading succeeds as GptOssForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export.\n", "hardware": null, "last_verified": "2026-08-25", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "expert_parallel": "2", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "2", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model unsloth/gpt-oss-20b-BF16 --megatron-path work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --hf-path work/model-verification/gpt-oss-20b/hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 2 --pp 2 --ep 2 --etp 1 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-megatron-to-hf-gpu", "expected_result": "The command exits successfully, exports the distributed Megatron checkpoint to the public unquantized BF16 Hugging Face layout, and writes a reloadable GptOssForCausalLM checkpoint with attention_bias=true. The export has 411 BF16 tensors, no missing or unexpected keys relative to unsloth/gpt-oss-20b-BF16, and all 20,914,757,184 parameters match that BF16 reference bitwise.\n", "hardware": null, "last_verified": "2026-08-09", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "expert_parallel": "2", "expert_tensor_parallel": "1", "pipeline_parallel": "2", "tensor_parallel": "2" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path openai/gpt-oss-20b --hf-revision 6cee5e81ee83917806bbde320786a8fb61efebee --megatron_model_path work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --tp 2 --pp 2 --ep 2 --etp 1 --prompt \"The capital of France is the city of\" --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-manual-forward-pass", "expected_result": "The pinned-revision one-step comparison exits successfully; the Hugging Face and Megatron next-token predictions match at token ID 12650 (\" Paris\"), and cosine similarity is 0.999913, above the 0.99 correlation gate. The maximum and mean absolute logit differences are 0.406250 and 0.051723, respectively; both are report-only diagnostic observations.\n", "hardware": null, "last_verified": "2026-08-09", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "expert_parallel": "2", "expert_tensor_parallel": "1", "pipeline_parallel": "2", "tensor_parallel": "2" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 scripts/inference/text_generation.py --hf_model_path unsloth/gpt-oss-20b-BF16 --megatron_model_path work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --prompt \"The capital of France is\" --max_new_tokens 32 --tp 2 --pp 2 --ep 2 --etp 1 --use-legacy-generation --attention-backend local --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-inference", "expected_result": "The command exits successfully after exactly 32 new tokens and prints this\nliteral completion, including its leading space and trailing space:\n Paris.\"\n # Test with a non-existent article\n article = fetch_article_by_id(999)\n assert article is None, \"Article with ID \n", "hardware": null, "last_verified": "2026-08-09", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "gpt_oss_20b_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe gpt_oss_20b_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-30b-a3b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load=null checkpoint.finetune=false checkpoint.save_optim=true checkpoint.save_rng=true --save_dir work/model-verification/gpt-oss-20b/pretrain-reference-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-pretrain-h100", "expected_result": "On 16x H100, the command completes exactly 100 bounded pretraining optimizer steps. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 10.90364 to 5.725391, all five metrics are recorded, the post-setup run_config.yaml persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers suitable for direct resume.\n", "hardware": "H100", "last_verified": "2026-08-09", "metrics": { "final_loss": 5.725391, "initial_loss": 10.90364, "last_10_steps_model_tflops_per_gpu_avg": 117.01, "last_10_steps_step_time_ms_avg": 22400.88, "last_10_steps_tokens_per_second_per_gpu_avg": 5851.199 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "gpt_oss_20b_sft_8gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_20b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"6cee5e81ee83917806bbde320786a8fb61efebee\"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-sft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-20b/sft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "gpt-oss-20b-sft-h100", "expected_result": "On 8x H100, the command completes exactly 100 full-SFT optimizer steps from the imported GPT-OSS 20B checkpoint with immutable Tulu3 data selection, assistant-token loss masking, and offline packing. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 2.130852 to 0.758708, all five metrics are recorded, packing is 98.56% efficient, and a complete 8-shard iter_0000100 full-model checkpoint is saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers.\n", "hardware": "H100", "last_verified": "2026-08-09", "metrics": { "final_loss": 0.758708, "initial_loss": 2.130852, "last_10_steps_model_tflops_per_gpu_avg": 109.05, "last_10_steps_step_time_ms_avg": 6618.85, "last_10_steps_tokens_per_second_per_gpu_avg": 4950.709 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "expert_parallel": "2", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "2", "total_gpus": 8 }, { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model unsloth/gpt-oss-20b-BF16 --megatron-path work/model-verification/gpt-oss-20b/sft-2k-checkpoints/iter_0000100 --hf-path work/model-verification/gpt-oss-20b/sft-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 2 --pp 2 --ep 2 --etp 1 --trust-remote-code", "./scripts/inference/infer.sh --task hf-inference --nodes 1 --gpus-per-node 1 --hf-model work/model-verification/gpt-oss-20b/sft-hf-export --prompt \"Name the capital of France and explain its role in one sentence.\" --max-new-tokens 45 --chat-template" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-sft-export-inference-h100", "expected_result": "The first command exits successfully and exports the final full-SFT checkpoint to a reloadable 9-shard Hugging Face BF16 checkpoint with 411 tensors and attention_bias=true. The second command strictly reloads the exported checkpoint with Transformers and runs deterministic greedy HF inference, stopped at EOS, and produced exactly 38 generated tokens within the 45-token bound with this literal completion: \"finalThe capital of France is Paris, and it serves as the political, economic, and cultural center of the country, hosting the headquarters of many national institutions and international organizations.\"\n", "hardware": "H100", "last_verified": "2026-08-09", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "context_parallel": "2", "gpus_per_node": "8", "nodes": "1", "recipe": "gpt_oss_20b_sft_8gpu_h100_bf16_32k_config", "sequence_length": "32768", "tensor_parallel": "4", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_20b_sft_8gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --tensor_model_parallel_size 4 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"6cee5e81ee83917806bbde320786a8fb61efebee\"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "offline" }, "entry_id": "gpt-oss-20b-sft-long-context-h100", "expected_result": "Completed 20 optimizer steps using the 32K SFT recipe, immutable Tulu3 data selection, offline packing, TP4, CP2, sequence parallelism, non-fused cross entropy, and full activation recompute. The run loaded 120 packed rows, recorded 99.79% packing efficiency with 83.333 source sequences per pack on average, produced finite loss from 2.235548 to 1.175326, and reported zero skipped and zero NaN iterations.\n", "hardware": "H100", "last_verified": "2026-08-10", "metrics": { "final_loss": 1.175326, "initial_loss": 2.235548, "last_10_steps_model_tflops_per_gpu_avg": 13.43, "last_10_steps_step_time_ms_avg": 306108.17, "last_10_steps_tokens_per_second_per_gpu_avg": 428.189 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "a8038f6d99d8ade33e1b715ba79bc9185b073e59", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "recipe": "gpt_oss_20b_peft_1gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 1 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gpt_oss_20b_peft_1gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"6cee5e81ee83917806bbde320786a8fb61efebee\"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-peft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-20b/peft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "gpt-oss-20b-peft-h100", "expected_result": "On 1x H100, the command completes exactly 100 LoRA PEFT optimizer steps from the imported GPT-OSS 20B checkpoint with the base model frozen, LoRA dim 32, alpha 32, dropout 0.0, target modules linear_qkv, linear_proj, linear_fc1, and linear_fc2, immutable Tulu3 data selection, assistant-token loss masking, and offline packing. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 2.129699 to 1.075515, all five metrics are recorded, packing is 98.56% efficient, and a complete single-shard iter_0000100 adapter checkpoint is saved with metadata, train state, run_config.yaml, and latest-checkpoint markers.\n", "hardware": "H100", "last_verified": "2026-08-09", "metrics": { "final_loss": 1.075515, "initial_loss": 2.129699, "last_10_steps_model_tflops_per_gpu_avg": 113.24, "last_10_steps_step_time_ms_avg": 50689.94, "last_10_steps_tokens_per_second_per_gpu_avg": 5171.519 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "6799f099712c961f15cf92d6bd88998f029d672d", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "gpt_oss_20b_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe gpt_oss_20b_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-30b-a3b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.ckpt_step=50 checkpoint.finetune=false --load_dir work/model-verification/gpt-oss-20b/pretrain-reference-checkpoints --save_dir work/model-verification/gpt-oss-20b/pretrain-resume-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "gpt-oss-20b-checkpoint-resume-h100", "expected_result": "The direct 16-GPU continuation restores optimizer, scheduler, data-order, and RNG state from iter_0000050, begins at step 51, and finishes exactly at step 100 in a distinct resumed root. All 50 resumed optimizer-step rows are present with finite loss, zero skipped iterations, and zero NaN iterations. Step-51 loss 6.426731 matches the uninterrupted reference exactly; step-100 loss 5.725306 differs from reference 5.725391 by 0.001484%, within the declared one-percent gate, so both sentinels match. A complete 16-shard iter_0000100 checkpoint is saved with metadata, train state, tokenizer artifacts, run_config.yaml, and latest-checkpoint markers.\n", "hardware": "H100", "last_verified": "2026-08-09", "metrics": { "final_loss": 5.725306, "initial_loss": 6.426731, "last_10_steps_model_tflops_per_gpu_avg": 118.42, "last_10_steps_step_time_ms_avg": 22130.16, "last_10_steps_tokens_per_second_per_gpu_avg": 5922.777 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "openai/gpt-oss-20b", "hf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "min_transformers_version": "5.8.0", "slug": "gpt-oss-20b", "source_card": "examples/model_verification_cards/gpt-oss-20b/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. GPT-OSS 20B H100 support verification has completed GPU HF-to-Megatron conversion, CPU and GPU Megatron-to-HF export, manual forward-pass correlation, deterministic Megatron inference, bounded pretraining, full SFT, SFT export/inference, long-context SFT, LoRA PEFT, and direct checkpoint resume; CPU import remains pending.\n", "title": "gpt_oss_20b" }, { "architecture": "Qwen3MoeForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08", "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "entries": [ { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-hf-to-megatron-cpu", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-hf-to-megatron-gpu", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-megatron-to-hf-cpu", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "unverified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-megatron-to-hf-gpu", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "unverified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-manual-forward-pass", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-inference", "expected_result": "This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "unverified", "variant": null, "workflow": "inference" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-pretrain-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-sft-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-sft-export-inference-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-sft-long-context-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-peft-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "bc09410353986ac7255ffd31fb2b720fb9007107", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-checkpoint-resume-h100", "expected_result": "This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "fbb7570cf7eec94fd2e6064454d84f7fad07fbfa", "command_topologies": [ { "gpus_per_node": "4", "nodes": "64", "recipe": "qwen3_235b_a22b_pretrain_256gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 256 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 64 --gpus-per-node 4 --recipe qwen3_235b_a22b_pretrain_256gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/qwen3-235b-a22b/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-pretrain-performance-gb300", "expected_result": "On exactly 256 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP4/CP1/EP32/ETP1, GBS/MBS 8192/2, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.75473 to 8.126335; the final ten steps average 14920.840 ms, 1300.440 TFLOP/s/GPU, and 8784.492 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB300", "last_verified": "2026-08-17", "metrics": { "final_loss": 8.126335, "initial_loss": 12.75473, "last_10_steps_model_tflops_per_gpu_avg": 1300.44, "last_10_steps_step_time_ms_avg": 14920.84, "last_10_steps_tokens_per_second_per_gpu_avg": 8784.492 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "0480586879f2513958fd8634fc529693ae13e536", "command_topologies": [ { "gpus_per_node": "4", "nodes": "64", "recipe": "qwen3_235b_a22b_pretrain_256gpu_gb200_fp8mx_config", "sequence_length": "4096", "total_gpus": 256 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 64 --gpus-per-node 4 --recipe qwen3_235b_a22b_pretrain_256gpu_gb200_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/qwen3-235b-a22b/gb200-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-235b-a22b-pretrain-performance-gb200", "expected_result": "On exactly 256 GB200s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP8/CP1/EP32/ETP1, VPP3, GBS/MBS 8192/1, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.75152 to 8.126326; the final ten steps average 17905.340 ms, 1083.660 TFLOP/s/GPU, and 7320.274 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB200", "last_verified": "2026-08-19", "metrics": { "final_loss": 8.126326, "initial_loss": 12.75152, "last_10_steps_model_tflops_per_gpu_avg": 1083.66, "last_10_steps_step_time_ms_avg": 17905.34, "last_10_steps_tokens_per_second_per_gpu_avg": 7320.274 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "Qwen/Qwen3-235B-A22B", "hf_revision": "8efa61729e24bd65b1d152b5ab5409052aa80e65", "min_transformers_version": "5.8.1", "slug": "qwen3-235b-a22b", "source_card": "examples/model_verification_cards/qwen3-235b-a22b/card.yaml", "summary": "Performance scope: pretrain_performance.GB200 and pretrain_performance.GB300 are verified with their tuned canonical 256-GPU MXFP8 recipes. Model-level and functional training workflows remain unverified in this card.\n", "title": "qwen3_235b_a22b" }, { "architecture": "Qwen3MoeForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.06", "bridge_commit": "aabb29d0ed15fc7ed881f0538352c6bd2d2fd52d", "entries": [ { "bridge_commit": "aabb29d0ed15fc7ed881f0538352c6bd2d2fd52d", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model Qwen/Qwen3-30B-A3B --megatron-path work/model-verification/qwen3-30b-a3b/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-hf-to-megatron-cpu", "expected_result": "The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with all 18,867 BF16 tensors matching the recorded HF revision bitwise.\n", "hardware": null, "last_verified": "2026-07-17", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "aabb29d0ed15fc7ed881f0538352c6bd2d2fd52d", "command_topologies": [ { "expert_parallel": "4", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "4", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model Qwen/Qwen3-30B-A3B --megatron-path work/model-verification/qwen3-30b-a3b/imported-megatron --torch-dtype bfloat16 --tp 4 --pp 2 --ep 4" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-hf-to-megatron-gpu", "expected_result": "The command exits successfully, creates iter_0000000, and all 18,867 BF16 tensors reload with keys, shapes, dtypes, and values exactly matching the recorded HF revision.\n", "hardware": null, "last_verified": "2026-07-17", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "aabb29d0ed15fc7ed881f0538352c6bd2d2fd52d", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model Qwen/Qwen3-30B-A3B --megatron-path work/model-verification/qwen3-30b-a3b/cpu-megatron/iter_0000000 --hf-path work/model-verification/qwen3-30b-a3b/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-megatron-to-hf-cpu", "expected_result": "The command exits successfully; all 18,867 exported BF16 tensors match the recorded HF revision bitwise, norm_topk_prob remains true, and the export reloads as Qwen3MoeForCausalLM without missing or unexpected keys.\n", "hardware": null, "last_verified": "2026-07-17", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "aabb29d0ed15fc7ed881f0538352c6bd2d2fd52d", "command_topologies": [ { "expert_parallel": "4", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "2", "tensor_parallel": "4", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model Qwen/Qwen3-30B-A3B --megatron-path work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --hf-path work/model-verification/qwen3-30b-a3b/hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 4 --pp 2 --ep 4" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-megatron-to-hf-gpu", "expected_result": "Strict export exits successfully, all 18,867 written BF16 tensors match the recorded HF revision bitwise, norm_topk_prob remains true, and Transformers reloads the output as Qwen3MoeForCausalLM without missing or unexpected keys.\n", "hardware": null, "last_verified": "2026-07-17", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "24f3dcde1fe27a1c3f1097b94d69b38e37a57415", "command_topologies": [ { "pipeline_parallel": "8", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path Qwen/Qwen3-30B-A3B --hf-revision ad44e777bcd18fa416d9da3bd8f70d33ebb85d39 --megatron_model_path work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --tp 1 --pp 8 --prompt \"The capital of France is the city of\"" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-manual-forward-pass", "expected_result": "The pinned-revision one-step comparison exits successfully; the Hugging Face and Megatron next-token predictions match at token ID 12095 (\" Paris\"), and cosine similarity is 0.999317, above the 0.99 correlation gate. The maximum and mean absolute logit differences are 0.593750 and 0.130064, respectively; both are report-only diagnostic observations.\n", "hardware": null, "last_verified": "2026-07-20", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "aabb29d0ed15fc7ed881f0538352c6bd2d2fd52d", "command_topologies": [ { "expert_parallel": "4", "pipeline_parallel": "2", "tensor_parallel": "4" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=8 examples/conversion/hf_to_megatron_generate_text.py --hf_model_path Qwen/Qwen3-30B-A3B --megatron_model_path work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --tp 4 --pp 2 --ep 4 --prompt \"The capital of France is\" --max_new_tokens 32" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-inference", "expected_result": "Two independent executions exit successfully after exactly 32 new tokens and print this byte-identical completion, including its leading space, \" Paris. The capital of the United Kingdom is London. The capital of the United States is Washington, D.C. The capital of Brazil is Bras\u00edlia. The\".\n", "hardware": null, "last_verified": "2026-07-17", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "5b9d9cf501193277e1ca47a99b97c640b8f39f90", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_pretrain_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-30b-a3b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model scheduler.lr_decay_iters=100 model.moe_router_force_load_balancing=false ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load=null validation.eval_iters=0 validation.eval_interval=0 dataset.random_seed=1234 dataset.num_workers=8 rng.seed=1234 dist.distributed_timeout_minutes=30 --save_dir work/model-verification/qwen3-30b-a3b/pretrain-convergence-v1-reference-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "cuda_graph": { "implementation": "transformer_engine", "scopes": [ "moe_router", "moe_preprocess" ] }, "moe_dispatcher": "hybridep" }, "entry_id": "qwen3-30b-a3b-pretrain-h100", "expected_result": "On 16x H100, the public alias resolves to the 16-GPU recipe and completes exactly 100 bounded RP2 optimizer steps with TP1/PP1/CP1/EP16/ETP1, DP16, SP off, GBS/MBS 1024/1, and 64-way gradient accumulation. Natural routing, HybridEP, and Transformer Engine CUDA graphs for moe_router and moe_preprocess remain active. Loss is finite from 12.41145 to 6.139116 with no skipped or NaN iterations, all five metrics are recorded, and complete iter_0000050 and iter_0000100 checkpoints are saved.\n", "hardware": "H100", "last_verified": "2026-07-19", "metrics": { "final_loss": 6.139116, "initial_loss": 12.41145, "last_10_steps_model_tflops_per_gpu_avg": 199.12, "last_10_steps_step_time_ms_avg": 30289.55, "last_10_steps_tokens_per_second_per_gpu_avg": 8654.602 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "487f563983f00b190a1a8322d663f317ad8e611c", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_functional_config", "sequence_length": "4096", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_functional_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-30b-a3b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model scheduler.lr_decay_iters=100 model.moe_router_force_load_balancing=false ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load=null checkpoint.save_optim=true checkpoint.save_rng=true checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.finetune=false validation.eval_iters=0 validation.eval_interval=0 dataset.random_seed=1234 dataset.num_workers=8 rng.seed=1234 dist.distributed_timeout_minutes=30 --save_dir work/model-verification/qwen3-30b-a3b/pretrain-mxfp8-gb200-reference-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "cuda_graph": { "implementation": "transformer_engine", "scopes": [ "moe_router", "moe_preprocess" ] }, "moe_dispatcher": "hybridep" }, "entry_id": "qwen3-30b-a3b-pretrain-gb200", "expected_result": "On 8x GB200, this support-verification workload completes exactly 100 bounded RP2 optimizer steps with TP1/PP1/CP1/EP8/ETP1, DP8, SP off, GBS/MBS 512/4, and 16-way gradient accumulation. MXFP8 compute, natural routing, HybridEP, Transformer Engine CUDA graphs for moe_router and moe_preprocess, communication overlap, functional safety checks, and MXFP8 parameter all-gather remain active. Loss is finite from 12.41293 to 6.183484 with no skipped or NaN iterations, all five metrics are recorded, and complete eight-shard iter_0000050 and iter_0000100 checkpoints are saved. Timing and throughput are support sanity checks, not cross-model convergence or tuned performance claims.\n", "hardware": "GB200", "last_verified": "2026-07-23", "metrics": { "final_loss": 6.183484, "initial_loss": 12.41293, "last_10_steps_model_tflops_per_gpu_avg": 487.88, "last_10_steps_step_time_ms_avg": 12362.34, "last_10_steps_tokens_per_second_per_gpu_avg": 21205.047 }, "precision": "fp8_mx", "source_pointer": "items.pretrain.GB200", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "619cc20bd3c7eca1dc84e8ea0f822307dfbd5cc3", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen3_30b_a3b_sft_config", "sequence_length": "2048", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_sft_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"ad44e777bcd18fa416d9da3bd8f70d33ebb85d39\"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --save_dir work/model-verification/qwen3-30b-a3b/sft-convergence-v1-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "qwen3-30b-a3b-sft-h100", "expected_result": "The immutable-revision 16-GPU run completes exactly 100 full-SFT steps at TP1/PP1/CP1/EP16/ETP1, DP16, SP off, GBS/MBS 32/1, and two-way gradient accumulation with natural routing. Pad-1 offline packing is 99.30% efficient, and the sampled 6,553,600 token slots contain 4,350,004 supervised tokens after label masking. LM loss is finite from 1.704380 to 0.9030643 with no skipped or NaN iterations, all five metrics are recorded, and the complete sixteen-shard iter_0000100 full-model checkpoint reloads successfully.\n", "hardware": "H100", "last_verified": "2026-07-21", "metrics": { "final_loss": 0.9030643, "initial_loss": 1.70438, "last_10_steps_model_tflops_per_gpu_avg": 64.02, "last_10_steps_step_time_ms_avg": 1206.32, "last_10_steps_tokens_per_second_per_gpu_avg": 3395.451 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "619cc20bd3c7eca1dc84e8ea0f822307dfbd5cc3", "command_topologies": [ { "expert_parallel": "16", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "2", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 16 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 2 --gpus-per-node 8 --hf-model Qwen/Qwen3-30B-A3B --megatron-path work/model-verification/qwen3-30b-a3b/sft-convergence-v1-checkpoints/iter_0000100 --hf-path work/model-verification/qwen3-30b-a3b/sft-convergence-v1-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 1 --pp 1 --ep 16 --etp 1", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/qwen3-30b-a3b/sft-convergence-v1-hf-export --prompt \"In one short sentence, explain why Paris is important to France.\" --max-new-tokens 32 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-sft-export-inference-h100", "expected_result": "The immutable-revision step-100 SFT checkpoint exports as 16 indexed BF16 shards with 18,867 weights, vocabulary size 151936, and embedding and output-head shapes [151936, 2048]. The index has the exact key set and total_size 61,064,245,248, equal to the serialized tensor bytes. Transformers strictly reloads with no errors, mismatches, missing, or unexpected keys. Two independent runs using greedy decoding produce byte-identical token IDs and exactly 32 new tokens with this literal completion: \"Paris is the capital city of France and is a major cultural, economic, and political center in Europe. It is home to many famous landmarks, such as the\".\n", "hardware": "H100", "last_verified": "2026-07-21", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "f3ae2767b5e18aeb67b726cd8d5f1db58216dcc9", "command_topologies": [ { "context_parallel": "2", "gpus_per_node": "8", "nodes": "2", "recipe": "qwen3_30b_a3b_sft_8gpu_h100_bf16_config", "sequence_length": "32768", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --context_parallel_size 2 -tp 8 -pp 1 -ep 8 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"ad44e777bcd18fa416d9da3bd8f70d33ebb85d39\"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=16 model.cp_comm_type=p2p model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "qwen3-30b-a3b-sft-long-context-h100", "expected_result": "The immutable-revision 16-GPU run completes exactly 20 Tulu3 SFT steps at sequence length 32768 with TP8/PP1/CP2/EP8/SP-on, DeepEP, and explicit pad-16 offline packing. LM loss is 1.645009 to 1.468103; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.28%, and the sampled training window contains 13,573,663 actual supervised tokens. PP=1 keeps tokens, labels, loss masks, and packed-sequence boundaries on one pipeline stage.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 1.468103, "initial_loss": 1.645009, "last_10_steps_model_tflops_per_gpu_avg": 26.12, "last_10_steps_step_time_ms_avg": 142663.71, "last_10_steps_tokens_per_second_per_gpu_avg": 459.374 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "619cc20bd3c7eca1dc84e8ea0f822307dfbd5cc3", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "recipe": "qwen3_30b_a3b_peft_4gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 4 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen3_30b_a3b_peft_4gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"ad44e777bcd18fa416d9da3bd8f70d33ebb85d39\"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --save_dir work/model-verification/qwen3-30b-a3b/peft-tp4-ep4-pad4-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "qwen3-30b-a3b-peft-h100", "expected_result": "The immutable-revision 4-GPU run completes exactly 100 PEFT steps at TP4/PP1/CP1/EP4/ETP1, DP1, SP on, GBS/MBS 32/1, and 32-way gradient accumulation with natural routing and DeepEP. Pad-4 offline packing is 99.39% efficient, and the sampled 6,553,600 token slots contain 4,332,480 supervised tokens after label masking. Only rank-8, alpha-16, zero-dropout LoRA on linear_qkv and linear_proj is trainable. LM loss is finite from 1.575987 to 1.113119 with no skipped or NaN iterations, all five metrics are recorded, and the complete four-shard iter_0000100 adapter checkpoint covers all 192 expected adapter entries.\n", "hardware": "H100", "last_verified": "2026-07-21", "metrics": { "final_loss": 1.113119, "initial_loss": 1.575987, "last_10_steps_model_tflops_per_gpu_avg": 13.84, "last_10_steps_step_time_ms_avg": 22347.64, "last_10_steps_tokens_per_second_per_gpu_avg": 733.142 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "5b9d9cf501193277e1ca47a99b97c640b8f39f90", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_pretrain_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-30b-a3b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model scheduler.lr_decay_iters=100 model.moe_router_force_load_balancing=false ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true validation.eval_iters=0 validation.eval_interval=0 dataset.random_seed=1234 dataset.num_workers=8 rng.seed=1234 dist.distributed_timeout_minutes=30 --load_dir work/model-verification/qwen3-30b-a3b/pretrain-convergence-v1-reference-checkpoints --save_dir work/model-verification/qwen3-30b-a3b/pretrain-convergence-v1-resumed-checkpoints --save_interval 50 checkpoint.ckpt_step=50 train.empty_unused_memory_level=2 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-checkpoint-resume-h100", "expected_result": "The command restores optimizer, scheduler, data-order, and RNG state from iter_0000050, begins at step 51, and finishes at step 100 in the distinct resumed root with finite losses and no skipped or NaN iterations. Releasing unused cached memory after optimizer steps is execution-only. Step-51 loss 6.989006 matches the uninterrupted reference exactly; step-100 loss 6.145390 differs from reference 6.139116 by 0.102197%, within the declared one-percent gate, so both sentinels match and all five metrics are recorded.\n", "hardware": "H100", "last_verified": "2026-07-19", "metrics": { "final_loss": 6.14539, "initial_loss": 6.989006, "last_10_steps_model_tflops_per_gpu_avg": 193.64, "last_10_steps_step_time_ms_avg": 31142.65, "last_10_steps_tokens_per_second_per_gpu_avg": 8417.524 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "487f563983f00b190a1a8322d663f317ad8e611c", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_functional_config", "sequence_length": "4096", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_functional_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-30b-a3b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model scheduler.lr_decay_iters=100 model.moe_router_force_load_balancing=false ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.save_optim=true checkpoint.save_rng=true checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.finetune=false validation.eval_iters=0 validation.eval_interval=0 dataset.random_seed=1234 dataset.num_workers=8 rng.seed=1234 dist.distributed_timeout_minutes=30 --load_dir work/model-verification/qwen3-30b-a3b/pretrain-mxfp8-gb200-reference-checkpoints --save_dir work/model-verification/qwen3-30b-a3b/pretrain-mxfp8-gb200-resumed-checkpoints --save_interval 50 checkpoint.ckpt_step=50 train.empty_unused_memory_level=2 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-checkpoint-resume-gb200", "expected_result": "The command restores optimizer, scheduler, data-order, and RNG state from iter_0000050, keeps the MXFP8 forward pre-hook active before the first resumed forward pass, begins at step 51, and finishes at step 100 in a distinct resumed root with finite losses and no skipped or NaN iterations. Step-51 loss 6.774675 matches the uninterrupted reference exactly; step-100 loss 6.187264 differs from reference 6.183484 by 0.061131%, within the declared one-percent gate. Both sentinels match, all five metrics are recorded, and a complete iter_0000100 checkpoint is saved.\n", "hardware": "GB200", "last_verified": "2026-07-23", "metrics": { "final_loss": 6.187264, "initial_loss": 6.774675, "last_10_steps_model_tflops_per_gpu_avg": 322.62, "last_10_steps_step_time_ms_avg": 18694.97, "last_10_steps_tokens_per_second_per_gpu_avg": 14022.167 }, "precision": "fp8_mx", "source_pointer": "items.checkpoint_resume.GB200", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "3a9559f01283190bf16a7a592c1e2c35d908c496", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_16gpu_h100_bf16_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_pretrain_16gpu_h100_bf16_config --mode pretrain --max_steps 50" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-pretrain-performance-h100", "expected_result": "On two nodes with 16x H100, the exact mock-data performance recipe completes exactly 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. HybridEP permute fusion uses 32 SMs and 64-token combine chunks, expert-parallel communication overlap is active with delayed weight-gradient compute disabled, and Transformer Engine CUDA graph capture completes for all 48 graphable layers with moe_router and moe_preprocess scopes. Over steps 41-50, the run averages at most 20.50 seconds per step and at least 295 model TFLOP/s/GPU, while peak allocated memory remains below 65 GiB/GPU.\n", "hardware": "H100", "last_verified": "2026-07-27", "metrics": { "final_loss": 8.145514, "initial_loss": 12.34643, "last_10_steps_model_tflops_per_gpu_avg": 299.352, "last_10_steps_step_time_ms_avg": 20147.29, "last_10_steps_tokens_per_second_per_gpu_avg": 13011.378 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.H100", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "6e1b47893e3c174e74a97fb1730c9f58387ab882", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_config --max_steps 50" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-pretrain-performance-gb200", "expected_result": "On two nodes with 8x GB200, the exact mock-data performance recipe completes exactly 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. The benchmark uses forced load balancing, HybridEP, MXFP8, and full-iteration CUDA graphs. Over steps 41-50, the run averages at most 7 seconds per step and at least 900 model TFLOP/s/GPU.\n", "hardware": "GB200", "last_verified": "2026-07-23", "metrics": { "final_loss": 8.112733, "initial_loss": 12.34754, "last_10_steps_model_tflops_per_gpu_avg": 927.68, "last_10_steps_step_time_ms_avg": 6501.12, "last_10_steps_tokens_per_second_per_gpu_avg": 40322.898 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "0480586879f2513958fd8634fc529693ae13e536", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 logger.save_config_filepath=work/model-verification/qwen3-30b-a3b/gb300-performance/ConfigContainer.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-pretrain-performance-gb300", "expected_result": "On exactly 8 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP8/ETP1, GBS/MBS 512/8, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.34753 to 8.125566; the final ten steps average 5854.910 ms, 1030.070 TFLOP/s/GPU, and 44773.361 tokens/s/GPU. The resolved configuration persists.\n", "hardware": "GB300", "last_verified": "2026-08-18", "metrics": { "final_loss": 8.125566, "initial_loss": 12.34753, "last_10_steps_model_tflops_per_gpu_avg": 1030.07, "last_10_steps_step_time_ms_avg": 5854.91, "last_10_steps_tokens_per_second_per_gpu_avg": 44773.361 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB300", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "fbb7570cf7eec94fd2e6064454d84f7fad07fbfa", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 --global_batch_size 512" ], "dimensions": { "global_batch_size": 512, "num_gpus": 8 }, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-pretrain-weak-scaling-gb300-8-gpu", "expected_result": "On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.\n", "hardware": "GB300", "last_verified": "2026-08-15", "metrics": { "final_loss": 8.131992, "initial_loss": 12.34753, "last_10_steps_model_tflops_per_gpu_avg": 1024.02, "last_10_steps_step_time_ms_avg": 5889.42, "last_10_steps_tokens_per_second_per_gpu_avg": 44511.004 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_weak_scaling.GB300.points[0]", "status": "verified", "variant": "8-gpu", "workflow": "pretrain_weak_scaling" }, { "bridge_commit": "fbb7570cf7eec94fd2e6064454d84f7fad07fbfa", "command_topologies": [ { "gpus_per_node": "4", "nodes": "8", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 32 } ], "commands": [ "./scripts/training/train.sh --nodes 8 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 --global_batch_size 2048" ], "dimensions": { "global_batch_size": 2048, "num_gpus": 32 }, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-pretrain-weak-scaling-gb300-32-gpu", "expected_result": "On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.\n", "hardware": "GB300", "last_verified": "2026-08-15", "metrics": { "final_loss": 8.131451, "initial_loss": 12.3471, "last_10_steps_model_tflops_per_gpu_avg": 1037.27, "last_10_steps_step_time_ms_avg": 5814.29, "last_10_steps_tokens_per_second_per_gpu_avg": 45086.158 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_weak_scaling.GB300.points[1]", "status": "verified", "variant": "32-gpu", "workflow": "pretrain_weak_scaling" }, { "bridge_commit": "fbb7570cf7eec94fd2e6064454d84f7fad07fbfa", "command_topologies": [ { "gpus_per_node": "4", "nodes": "32", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 128 } ], "commands": [ "./scripts/training/train.sh --nodes 32 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 --global_batch_size 8192" ], "dimensions": { "global_batch_size": 8192, "num_gpus": 128 }, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-pretrain-weak-scaling-gb300-128-gpu", "expected_result": "On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.\n", "hardware": "GB300", "last_verified": "2026-08-15", "metrics": { "final_loss": 8.126566, "initial_loss": 12.34679, "last_10_steps_model_tflops_per_gpu_avg": 1034.18, "last_10_steps_step_time_ms_avg": 5831.72, "last_10_steps_tokens_per_second_per_gpu_avg": 44951.404 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_weak_scaling.GB300.points[2]", "status": "verified", "variant": "128-gpu", "workflow": "pretrain_weak_scaling" }, { "bridge_commit": "fbb7570cf7eec94fd2e6064454d84f7fad07fbfa", "command_topologies": [ { "gpus_per_node": "4", "nodes": "64", "recipe": "qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config", "sequence_length": "4096", "total_gpus": 256 } ], "commands": [ "./scripts/training/train.sh --nodes 64 --gpus-per-node 4 --recipe qwen3_30b_a3b_pretrain_8gpu_gb300_fp8mx_config --mode pretrain --max_steps 50 --seq_length 4096 --global_batch_size 16384" ], "dimensions": { "global_batch_size": 16384, "num_gpus": 256 }, "enabled_features": {}, "entry_id": "qwen3-30b-a3b-pretrain-weak-scaling-gb300-256-gpu", "expected_result": "On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.\n", "hardware": "GB300", "last_verified": "2026-08-15", "metrics": { "final_loss": 8.129371, "initial_loss": 12.34698, "last_10_steps_model_tflops_per_gpu_avg": 1036.49, "last_10_steps_step_time_ms_avg": 5818.62, "last_10_steps_tokens_per_second_per_gpu_avg": 45052.607 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_weak_scaling.GB300.points[3]", "status": "verified", "variant": "256-gpu", "workflow": "pretrain_weak_scaling" } ], "hf_id": "Qwen/Qwen3-30B-A3B", "hf_revision": "ad44e777bcd18fa416d9da3bd8f70d33ebb85d39", "min_transformers_version": "5.8.1", "slug": "qwen3-30b-a3b", "source_card": "examples/model_verification_cards/qwen3-30b-a3b/card.yaml", "summary": "Performance scope: pretrain_performance.H100 and pretrain_performance.GB200 and pretrain_performance.GB300 use tuned canonical performance recipes; timing and throughput metrics from functional training items remain sanity checks rather than optimized performance results. Weak-scaling scope: pretrain_weak_scaling.GB300 records measured scaling from 8 through 256 GPUs with constant token slots per GPU per step. Qwen3-30B-A3B support verification covers conversion, inference, and training.\n", "title": "qwen3_30b_a3b" }, { "architecture": "Qwen3ForCausalLM", "base_container": "nvcr.io/nvidia/pytorch:26.04-py3", "bridge_commit": "5c56eab34c540fad08544c38b7cc39d662fb7475", "entries": [ { "bridge_commit": "5c56eab34c540fad08544c38b7cc39d662fb7475", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model Qwen/Qwen3-8B --megatron-path work/model-verification/qwen3-8b/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-hf-to-megatron-cpu", "expected_result": "The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with all 399 HF tensors matching the recorded HF revision exactly in keys, shapes, dtypes, and values.\n", "hardware": null, "last_verified": "2026-07-16", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "5c56eab34c540fad08544c38b7cc39d662fb7475", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "tensor_parallel": "4", "total_gpus": 4 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --hf-model Qwen/Qwen3-8B --megatron-path work/model-verification/qwen3-8b/imported-megatron --torch-dtype bfloat16 --tp 4" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-hf-to-megatron-gpu", "expected_result": "The command exits successfully, creates iter_0000000, and the checkpoint reloads at TP=4 with weights exactly matching the recorded HF revision.\n", "hardware": null, "last_verified": "2026-07-16", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "5c56eab34c540fad08544c38b7cc39d662fb7475", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model Qwen/Qwen3-8B --megatron-path work/model-verification/qwen3-8b/cpu-megatron/iter_0000000 --hf-path work/model-verification/qwen3-8b/cpu-hf-export" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-megatron-to-hf-cpu", "expected_result": "The command exits successfully; all 399 exported tensors match the recorded HF revision exactly, and the export reloads on CPU as Qwen3ForCausalLM.\n", "hardware": null, "last_verified": "2026-07-16", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "5c56eab34c540fad08544c38b7cc39d662fb7475", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "tensor_parallel": "4", "total_gpus": 4 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --hf-model Qwen/Qwen3-8B --megatron-path work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --hf-path work/model-verification/qwen3-8b/hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 4" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-megatron-to-hf-gpu", "expected_result": "Strict export exits successfully and the Hugging Face output reloads with AutoModelForCausalLM as Qwen3ForCausalLM.\n", "hardware": null, "last_verified": "2026-07-16", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "5c56eab34c540fad08544c38b7cc39d662fb7475", "command_topologies": [ { "tensor_parallel": "4" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=4 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path Qwen/Qwen3-8B --megatron_model_path work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --tp 4 --prompt \"The capital of France is the city of\"" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-manual-forward-pass", "expected_result": "The one-step comparison of the eight-token prompt exits successfully; the Hugging Face and Megatron next-token predictions match at token ID 12095 (\" Paris\"), and cosine similarity is 0.999969, above the 0.99 correlation gate. This historical result predates explicit helper revision pinning and is retained against the recorded immutable HF revision. The maximum and mean absolute logit differences are 0.187500 and 0.030649, respectively; both are report-only diagnostic observations.\n", "hardware": null, "last_verified": "2026-07-17", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "5c56eab34c540fad08544c38b7cc39d662fb7475", "command_topologies": [ { "tensor_parallel": "4" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_text.py --hf_model_path Qwen/Qwen3-8B --megatron_model_path work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --tp 4 --prompt \"The capital of France is\" --max_new_tokens 32" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-inference", "expected_result": "Two independent executions exit successfully after exactly 32 generation steps and print this byte-identical completion, including its leading space, \" Paris. The capital of Italy is Rome. The capital of Spain is Madrid. The capital of Germany is Berlin. The capital of the Netherlands is Amsterdam. The\".\n", "hardware": null, "last_verified": "2026-07-16", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "619cc20bd3c7eca1dc84e8ea0f822307dfbd5cc3", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen3_8b_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_8b_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-8b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 dist.distributed_timeout_minutes=30 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true checkpoint.load=null --save_dir work/model-verification/qwen3-8b/pretrain-reference-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-pretrain-h100", "expected_result": "The uninterrupted 16-GPU bounded RP2 run completes exactly 100 steps at TP1/PP1/CP1, DP16, GBS/MBS 1024/1, and 64-way gradient accumulation. It reaches peak learning rate at step 40 and completes cosine decay at step 100. LM loss is finite from 12.73617 to 6.190218 with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved.\n", "hardware": "H100", "last_verified": "2026-07-21", "metrics": { "final_loss": 6.190218, "initial_loss": 12.73617, "last_10_steps_model_tflops_per_gpu_avg": 512.7, "last_10_steps_step_time_ms_avg": 25070.32, "last_10_steps_tokens_per_second_per_gpu_avg": 10456.348 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "f3ae2767b5e18aeb67b726cd8d5f1db58216dcc9", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "recipe": "qwen3_8b_sft_4gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 4 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen3_8b_sft_4gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"b968826d9c46dd6066d109eabc6255188de91218\"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/qwen3-8b/sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "qwen3-8b-sft-h100", "expected_result": "Immutable Tulu3 pad-1 offline packing is 99.30% efficient. Full SFT uses DP=1 with 32 gradient-accumulation steps and reaches step 100 with the four recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, the sampled assistant-only loss masks contain 4,350,004 supervised tokens. The complete four-shard iter_0000100 full-model checkpoint is saved.\n", "hardware": "H100", "last_verified": "2026-07-19", "metrics": { "final_loss": 0.9733383, "initial_loss": 1.657044, "last_10_steps_model_tflops_per_gpu_avg": 101.63, "last_10_steps_step_time_ms_avg": 7424.24, "last_10_steps_tokens_per_second_per_gpu_avg": 2206.825 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "f3ae2767b5e18aeb67b726cd8d5f1db58216dcc9", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "tensor_parallel": "4", "total_gpus": 4 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --hf-model Qwen/Qwen3-8B --megatron-path work/model-verification/qwen3-8b/sft-checkpoints/iter_0000100 --hf-path work/model-verification/qwen3-8b/sft-hf-export --torch-dtype bfloat16 --export-weight-dtype bfloat16 --distributed-save --tp 4", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/qwen3-8b/sft-hf-export --prompt \"Name the capital of France and explain its role in one sentence.\" --max-new-tokens 45 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-sft-export-inference-h100", "expected_result": "The current full-SFT checkpoint exports as five indexed BF16 shards with 399 weights, config vocabulary size 151936, and embedding and output-head shapes [151936, 4096]. Transformers natively reloads all 399 weights, and two independent runs using greedy generation produce byte-identical token IDs and exactly 45 new tokens with this literal completion, including the escaped blank line: \"The capital of France is Paris. It is the political, economic, and cultural center of France and serves as the seat of the French government. \u2705\\n\\nParis is also a major tourist destination, known for its iconic landmarks\".\n", "hardware": "H100", "last_verified": "2026-07-19", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "f3ae2767b5e18aeb67b726cd8d5f1db58216dcc9", "command_topologies": [ { "context_parallel": "2", "gpus_per_node": "8", "nodes": "1", "recipe": "qwen3_8b_sft_32k_config", "sequence_length": "32768", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe qwen3_8b_sft_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"b968826d9c46dd6066d109eabc6255188de91218\"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=8 model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "offline" }, "entry_id": "qwen3-8b-sft-long-context-h100", "expected_result": "The immutable-revision 8-GPU run completes exactly 20 Tulu3 SFT steps at sequence length 32768 with recipe-owned TP4/PP1/CP2/SP-on, GBS/MBS 8/1, and explicit pad-8 offline packing. LM loss is 1.611622 to 1.490618; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.97%, and the sampled training window contains 3,444,917 actual supervised tokens.\n", "hardware": "H100", "last_verified": "2026-07-20", "metrics": { "final_loss": 1.490618, "initial_loss": 1.611622, "last_10_steps_model_tflops_per_gpu_avg": 34.08, "last_10_steps_step_time_ms_avg": 71668.78, "last_10_steps_tokens_per_second_per_gpu_avg": 457.214 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "f3ae2767b5e18aeb67b726cd8d5f1db58216dcc9", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "recipe": "qwen3_8b_peft_1gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 1 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe qwen3_8b_peft_1gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split=\"train[:10000]\"' 'dataset.hf_dataset.load_kwargs={revision:\"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be\"}' '++tokenizer.hf_tokenizer_kwargs.revision=\"b968826d9c46dd6066d109eabc6255188de91218\"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/qwen3-8b/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "qwen3-8b-peft-h100", "expected_result": "Immutable Tulu3 pad-4 offline packing is 99.39% efficient. The 100 LoRA steps use DP=1 with 32 gradient-accumulation steps and complete with the five recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, the sampled assistant-only loss masks contain 4,332,480 supervised tokens. The complete single-shard iter_0000100 adapter checkpoint is saved.\n", "hardware": "H100", "last_verified": "2026-07-19", "metrics": { "final_loss": 1.170038, "initial_loss": 1.527745, "last_10_steps_model_tflops_per_gpu_avg": 387.43, "last_10_steps_step_time_ms_avg": 7774.79, "last_10_steps_tokens_per_second_per_gpu_avg": 8429.295 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "619cc20bd3c7eca1dc84e8ea0f822307dfbd5cc3", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen3_8b_pretrain_16gpu_h100_bf16_config", "sequence_length": "4096", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_8b_pretrain_16gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01\"],null]' dataset.path_to_cache=work/cache/qwen3-8b/rp2 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer.model rng.seed=1234 dataset.random_seed=1234 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 dist.distributed_timeout_minutes=30 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --load_dir work/model-verification/qwen3-8b/pretrain-reference-checkpoints --save_dir work/model-verification/qwen3-8b/pretrain-resumed-from-reference-checkpoints --save_interval 50 checkpoint.ckpt_step=50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8b-checkpoint-resume-h100", "expected_result": "The direct 16-GPU continuation restores optimizer, scheduler, data-order, and RNG state from iter_0000050, begins at step 51, and finishes exactly at step 100 in a distinct output root at TP1/PP1/CP1, DP16, GBS/MBS 1024/1, and 64-way gradient accumulation. All 50 losses are finite with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and a complete 16-shard iter_0000100 checkpoint is saved. Step-51 resumed/reference loss is 7.015641/7.015641. Step-100 resumed/reference loss is 6.190785/6.190218, an absolute difference of 0.000567. This difference does not meet the card's previous 1e-6-relative threshold; this verified result uses the model-verification skill's standard 1%-relative gate, which was fixed for this 16-GPU execution layout before the run.\n", "hardware": "H100", "last_verified": "2026-07-21", "metrics": { "final_loss": 6.190785, "initial_loss": 7.015641, "last_10_steps_model_tflops_per_gpu_avg": 514.24, "last_10_steps_step_time_ms_avg": 24995.61, "last_10_steps_tokens_per_second_per_gpu_avg": 10487.602 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "Qwen/Qwen3-8B", "hf_revision": "b968826d9c46dd6066d109eabc6255188de91218", "min_transformers_version": "5.8.1", "slug": "qwen3-8b", "source_card": "examples/model_verification_cards/qwen3-8b/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Qwen3-8B support verification covers conversion, inference, and training.\n", "title": "qwen3_8b" }, { "architecture": "Qwen3_5MoeForConditionalGeneration", "base_container": "nvcr.io/nvidia/nemo:26.06", "bridge_commit": "5db2290dea4887a2ca9072c0d554707bf272945d", "entries": [ { "bridge_commit": "58b11d7e122b6d7da57d1b5f264fe7fa61e393ec", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --gpus-per-node 1 --mem 512G --hf-model Qwen/Qwen3.6-35B-A3B --hf-revision 995ad96eacd98c81ed38be0c5b274b04031597b0 --megatron-path work/model-verification/qwen3.6-35b-a3b/cpu-megatron" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-hf-to-megatron-cpu", "expected_result": "The pinned-revision import exited successfully in 14m39s and created iter_0000000. Model weights remained on CPU; exactly one shared runtime GPU was required because Transformer Engine constructs the Qwen vision modules through CUDA even for CPU-weight conversion. The resulting checkpoint was reloadable by the subsequent verified CPU export.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "5db2290dea4887a2ca9072c0d554707bf272945d", "command_topologies": [ { "expert_parallel": "8", "gpus_per_node": "8", "nodes": "2", "tensor_parallel": "2", "total_gpus": 16 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 2 --gpus-per-node 8 --hf-model Qwen/Qwen3.6-35B-A3B --hf-revision 995ad96eacd98c81ed38be0c5b274b04031597b0 --megatron-path work/model-verification/qwen3.6-35b-a3b/imported-megatron --tp 2 --ep 8" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-hf-to-megatron-gpu", "expected_result": "The 16-H100 pinned-revision import exited successfully at TP2/PP1/EP8/ETP1 and created a 16-shard iter_0000000 distributed checkpoint. Subsequent verified GPU export and inference workloads reloaded this checkpoint successfully.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "72d554863cd4dbcacd26abda92054afd0506e668", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --gpus-per-node 1 --mem 512G --hf-model Qwen/Qwen3.6-35B-A3B --hf-revision 995ad96eacd98c81ed38be0c5b274b04031597b0 --megatron-path work/model-verification/qwen3.6-35b-a3b/cpu-megatron/iter_0000000 --hf-path work/model-verification/qwen3.6-35b-a3b/cpu-hf-export" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-megatron-to-hf-cpu", "expected_result": "Export exited successfully in 6m42s with weights on CPU and exactly one shared runtime GPU for Transformer Engine vision-module construction. Transformers strictly reloaded all 1,045 BF16 tensors in 26 indexed shards as Qwen3_5MoeForConditionalGeneration. The exported config preserves 40 language layers, 256 experts, top-8 routing, 27 vision layers, and one MTP layer with dedicated embeddings disabled.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "5db2290dea4887a2ca9072c0d554707bf272945d", "command_topologies": [ { "expert_parallel": "8", "gpus_per_node": "8", "nodes": "2", "tensor_parallel": "2", "total_gpus": 16 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 2 --gpus-per-node 8 --hf-model Qwen/Qwen3.6-35B-A3B --hf-revision 995ad96eacd98c81ed38be0c5b274b04031597b0 --megatron-path work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --hf-path work/model-verification/qwen3.6-35b-a3b/hf-export --tp 2 --ep 8" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-megatron-to-hf-gpu", "expected_result": "The 16-H100 distributed export exited successfully and wrote 26 indexed shards containing 1,045 BF16 tensors and 71,903,645,408 serialized tensor bytes. Transformers independently reloaded the persisted export as Qwen3_5MoeForConditionalGeneration with empty missing, unexpected, mismatched, and error sets.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "ac39e582036c4f120257735bf22d71b8d7a1cddb", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "8", "total_gpus": 8 } ], "commands": [ "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 8 --task model-comparison --hf_model_path Qwen/Qwen3.6-35B-A3B --hf-revision 995ad96eacd98c81ed38be0c5b274b04031597b0 --model_class Qwen3_5MoeForConditionalGeneration --megatron_model_path work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --pp 8 --image_path work/data/medpix/verification.png --prompt \"What was observed in the imaging?\"" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-manual-forward-pass", "expected_result": "The input was MedPix-VQA revision 69eb219d477ab8699296da1fd6b89beb768042d3, validation row 0, case MPX1095. Its 256-by-256 source image has SHA-256 d5852239309244feb646739dfe7460048c596380abe89def0a84d8a62778b7b1. The Qwen processor produced 83 input tokens and a 256-by-1536 pixel tensor. The pinned HF model and imported Megatron checkpoint had next-token predictions that matched at token 760 ('The'). Cosine similarity: 0.999156; maximum and mean absolute logit differences were 0.484375 and 0.082185, respectively.\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "72d554863cd4dbcacd26abda92054afd0506e668", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "8", "total_gpus": 8 } ], "commands": [ "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 8 --task vlm-generation --hf_model_path Qwen/Qwen3.6-35B-A3B --hf-revision 995ad96eacd98c81ed38be0c5b274b04031597b0 --megatron_model_path work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --pp 8 --image_path work/data/medpix/verification.png --prompt \"What was observed in the imaging?\" --max_new_tokens 32" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-inference", "expected_result": "Two independent runs used greedy decoding, loaded the imported checkpoint, consumed MedPix-VQA revision 69eb219d477ab8699296da1fd6b89beb768042d3 validation row 0 (case MPX1095; source-image SHA-256 d5852239309244feb646739dfe7460048c596380abe89def0a84d8a62778b7b1) and its question through the Qwen processor, and exited successfully with byte-identical 32-token completions. Neither run emitted EOS before the safety bound. The exact completion, represented with JSON newline escapes, was \"The user wants me to identify the findings in the provided MRI image.\\n\\n1. **Analyze the image:**\\n * **Modality:**\".\n", "hardware": null, "last_verified": "2026-07-23", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "52601b0382f869c9ca14109acdcdef77cc4fa61a", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "qwen35_vl_35b_a3b_pretrain_mock_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_pretrain_mock_config --dataset energon --deterministic --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 --save_dir work/model-verification/qwen3.6-35b-a3b/datacomp-deterministic-reference/checkpoints --save_interval 10 dataset.path=work/data/datacomp/energon dataset.task_encoder.hf_processor_path=Qwen/Qwen3.6-35B-A3B dataset.task_encoder.hf_processor_revision=995ad96eacd98c81ed38be0c5b274b04031597b0 dataset.do_validation=false scheduler.lr_decay_iters=20 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 checkpoint.load=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/datacomp-deterministic-reference/resolved-config.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-pretrain-h100", "expected_result": "At the recorded bridge commit, this bounded projection-pretraining support run used the official DataComp download pipeline at repository commit 4a8df1992566, the DataComp-1B metadata at revision 086ebeee20d4, and img2dataset 1.40.0. The Energon dataset contains 519,827 training and 5,173 validation image-caption samples; each training record maps an image plus \"Describe this image.\" to its caption. The deterministic 20-step reference consumed 10,240 training samples without wrapping. It warm-started the imported Qwen3.6 checkpoint, froze the language and vision towers, and trained the vision projection at TP1/PP1/CP1/EP8/ETP1, DP8, MBS1/GBS512, and 64-way gradient accumulation with one-layer full-uniform recompute. Loss was finite from 3.610127 to 2.261159 with zero skipped or NaN iterations; steps 11-20 averaged 330,730.470 ms and 18.090 TFLOP/s/GPU. Complete 72,366,371,530-byte eight-shard model/optimizer/RNG checkpoints and eight nonempty Energon rank states were saved at both steps 10 and 20, and the process exited successfully in 1h54m51s. This verifies the DataComp workflow and bounded Qwen VLM pretraining support, not canonical DataComp/CLIP convergence.\n", "hardware": "H100", "last_verified": "2026-07-28", "metrics": { "final_loss": 2.261159, "initial_loss": 3.610127, "last_10_steps_model_tflops_per_gpu_avg": 18.09, "last_10_steps_step_time_ms_avg": 330730.47, "last_10_steps_tokens_per_second_per_gpu_avg": 792.621 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "7f9b59730d17a255d46ff02e3795546f0284f748", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen35_vl_35b_a3b_sft_16gpu_h100_bf16_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_sft_16gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false dataset.pad_to_max_length=true scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.6-35b-a3b/sft-support-v11/checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/sft-support-v11/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "qwen3-6-35b-a3b-sft-h100", "expected_result": "The exact 16-H100 command completed all 100 MedPix SFT steps at TP1/PP2/CP1/EP8/ETP1, MBS1/GBS32, natural expert routing, HybridEP, one-layer full-uniform recompute, graph-free language and vision stacks, FP32 optimizer state, and Transformer Engine fused cross entropy. Loss was finite from 2.211317 to 1.006886 with zero skipped or NaN iterations. Steps 91-100 averaged 3,268.140 ms and 57.690 model TFLOP/s/GPU. Step-50 and step-100 saves both completed; the final checkpoint contains 20 nonempty files, including 16 distributed shards, totaling 509,636,716,209 bytes, and the tracker selects iteration 100.\n", "hardware": "H100", "last_verified": "2026-08-05", "metrics": { "final_loss": 1.006886, "initial_loss": 2.211317, "last_10_steps_model_tflops_per_gpu_avg": 57.69, "last_10_steps_step_time_ms_avg": 3268.14, "last_10_steps_tokens_per_second_per_gpu_avg": 2506.625 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "5db2290dea4887a2ca9072c0d554707bf272945d", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "qwen3-6-35b-a3b-sft-gb200", "expected_result": "On 8x GB200, the shared Qwen3.5/Qwen3.6-VL functional recipe completed exactly 100 MedPix SFT steps at TP1/PP1/CP1/EP8/ETP1, MBS1/GBS32, natural routing, HybridEP, no activation recompute, and BF16 gradient reduction. Loss was finite from 2.212866 to 0.9978729 with zero skipped or NaN iterations. Steps 91-100 averaged 9,899.640 ms and 38.180 model TFLOP/s/GPU. The recipe intentionally disables checkpoint output for this bounded functional run; reload/export coverage remains represented by the H100 SFT and sft_export_inference items.\n", "hardware": "GB200", "last_verified": "2026-07-30", "metrics": { "final_loss": 0.9978729, "initial_loss": 2.212866, "last_10_steps_model_tflops_per_gpu_avg": 38.18, "last_10_steps_step_time_ms_avg": 9899.64, "last_10_steps_tokens_per_second_per_gpu_avg": 1655.01 }, "precision": "bf16", "source_pointer": "items.sft.GB200", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "7f9b59730d17a255d46ff02e3795546f0284f748", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 }, { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --gpus-per-node 1 --mem 512G --hf-model Qwen/Qwen3.6-35B-A3B --hf-revision 995ad96eacd98c81ed38be0c5b274b04031597b0 --megatron-path work/model-verification/qwen3.6-35b-a3b/sft-support-v11/checkpoints/iter_0000100 --hf-path work/model-verification/qwen3.6-35b-a3b/sft-support-v11/hf-export", "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 1 --mem 256G --task hf-inference --hf-model work/model-verification/qwen3.6-35b-a3b/sft-support-v11/hf-export --prompt \"What was observed in the imaging?\" --image work/data/medpix/verification.png --max-new-tokens 32 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-sft-export-inference-h100", "expected_result": "CPU export from the exact step-100 SFT checkpoint completed successfully, converted 21,802 mappings, and reported that every source tensor was written. The unchanged HF path contains 34 nonempty files totaling 71,923,885,897 bytes, including all 26 indexed safetensors shards. A subsequent shared one-H100 run strictly reloaded all 1,026 tensors and completed the bounded processor-native MedPix prompt. It generated exactly 32 tokens at the 32-token maximum. The literal completion was: \"The imaging shows CT: Hyperdense mass in the right cerebellar hemisphere with surrounding edema.\\n\\nMRI: Right cerebellar hemisphere mass with mixed\".\n", "hardware": "H100", "last_verified": "2026-08-05", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "5ca8f9fb536808d49477b58b33538f5f6629ed3f", "command_topologies": [ { "gpus_per_node": "8", "nodes": "4", "recipe": "qwen35_vl_35b_a3b_sft_long_context_32gpu_h100_bf16_config", "total_gpus": 32 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_sft_long_context_32gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false scheduler.lr_decay_iters=20 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 checkpoint.load=null checkpoint.save=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/medpix-long-context/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "in_batch" }, "entry_id": "qwen3-6-35b-a3b-sft-long-context-h100", "expected_result": "The pinned 32-H100 run completed exactly 20 full-SFT steps at TP1/PP4/CP2/EP8/ETP1, dense DP4 and expert DP1, MBS2/GBS512, and 64-way gradient accumulation. It used 8192-token processor-native deferred in-batch packing with pad-to-multiple-of-4, fixed 448-by-448 MedPix images, per-token loss, and one-layer full-uniform activation recompute. The persisted post-setup config confirms every setting. Loss was finite from 1.997653 to 0.997894 with zero skipped or NaN iterations. After the compile-heavy first step, steps 11-20 averaged 36,847.110 ms and 84.830 TFLOP/s/GPU; the process exited successfully.\n", "hardware": "H100", "last_verified": "2026-07-28", "metrics": { "final_loss": 0.997894, "initial_loss": 1.997653, "last_10_steps_model_tflops_per_gpu_avg": 84.83, "last_10_steps_step_time_ms_avg": 36847.11, "last_10_steps_tokens_per_second_per_gpu_avg": 3557.185 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "19b2647fe9495dd1497e0400a760abf8ca541309", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen35_vl_35b_a3b_peft_16gpu_h100_bf16_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_peft_16gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false dataset.pad_to_max_length=true scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.6-35b-a3b/peft-support-v5/checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/peft-support-v5/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { "cuda_graph": { "implementation": "transformer_engine", "scopes": [ "attn", "mlp", "moe_router", "moe_preprocess" ] }, "moe_dispatcher": "hybridep" }, "entry_id": "qwen3-6-35b-a3b-peft-h100", "expected_result": "The exact 16-H100 command completed all 100 MedPix LoRA steps at TP1/PP2/CP1/EP8/ETP1, MBS1/GBS32, natural expert routing, HybridEP, scoped Transformer Engine CUDA graphs, and no activation recompute. The recipe disables dispatch-backward/expert-wgrad overlap because LoRA-wrapped expert linears do not expose Megatron Core's backward_dw hook. Loss was finite from 2.212696 to 1.270382 with zero skipped or NaN iterations. Steps 91-100 averaged 1,714.300 ms and 109.940 model TFLOP/s/GPU. The step-100 checkpoint contains 20 nonempty files, including 16 distributed shards, totaling 5,004,722,270 bytes.\n", "hardware": "H100", "last_verified": "2026-08-05", "metrics": { "final_loss": 1.270382, "initial_loss": 2.212696, "last_10_steps_model_tflops_per_gpu_avg": 109.94, "last_10_steps_step_time_ms_avg": 1714.3, "last_10_steps_tokens_per_second_per_gpu_avg": 4778.627 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "5db2290dea4887a2ca9072c0d554707bf272945d", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "hybridep" }, "entry_id": "qwen3-6-35b-a3b-peft-gb200", "expected_result": "On the same 8x GB200 topology and MedPix schedule as the controlled SFT item, LoRA completed exactly 100 finite steps with zero skipped or NaN iterations. The persisted configs differ only in the evidence-output path, the intended PEFT LR/min-LR, and the LoRA block; model, data, topology, batch, schedule, and safety settings match. Step-1 loss 2.214827 differs from SFT's 2.212866 by only 0.001961, or 0.088618%, ruling out an initial-forward configuration mismatch. The later loss difference reflects LoRA's trainable parameter set and learning rate. Steps 91-100 averaged 5,511.530 ms and 68.410 model TFLOP/s/GPU.\n", "hardware": "GB200", "last_verified": "2026-07-30", "metrics": { "final_loss": 1.254138, "initial_loss": 2.214827, "last_10_steps_model_tflops_per_gpu_avg": 68.41, "last_10_steps_step_time_ms_avg": 5511.53, "last_10_steps_tokens_per_second_per_gpu_avg": 2972.677 }, "precision": "bf16", "source_pointer": "items.peft.GB200", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "52601b0382f869c9ca14109acdcdef77cc4fa61a", "command_topologies": [ { "gpus_per_node": "8", "nodes": "1", "recipe": "qwen35_vl_35b_a3b_pretrain_mock_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_pretrain_mock_config --dataset energon --deterministic --load_dir work/model-verification/qwen3.6-35b-a3b/datacomp-deterministic-reference/checkpoints --max_steps 20 --warmup_iters 2 --save_dir work/model-verification/qwen3.6-35b-a3b/datacomp-deterministic-resume/checkpoints --save_interval 10 dataset.path=work/data/datacomp/energon dataset.task_encoder.hf_processor_path=Qwen/Qwen3.6-35B-A3B dataset.task_encoder.hf_processor_revision=995ad96eacd98c81ed38be0c5b274b04031597b0 dataset.do_validation=false scheduler.lr_decay_iters=20 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 checkpoint.ckpt_step=10 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/datacomp-deterministic-resume/resolved-config.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-checkpoint-resume-h100", "expected_result": "A fresh process loaded the exact reference step-10 checkpoint without a fallback warm start, restored model, optimizer, scheduler, RNG, and all eight Energon data-order states, entered the loop at iteration 10, and completed steps 11-20 in a distinct output root with zero skipped or NaN iterations. Step-11 loss 2.324095 matches the uninterrupted reference exactly. Step-20 loss 2.262711 differs from reference 2.261159 by 0.001552, or 0.068637%, within the declared one-percent gate; every intermediate resumed step was also within 0.1% of its reference. The resumed process exited successfully in 59m52s and saved a complete 72,366,371,502-byte eight-shard model/optimizer/RNG checkpoint plus eight nonempty Energon rank states at step 20, with the tracker selecting iteration 20.\n", "hardware": "H100", "last_verified": "2026-07-28", "metrics": { "final_loss": 2.262711, "initial_loss": 2.324095, "last_10_steps_model_tflops_per_gpu_avg": 17.65, "last_10_steps_step_time_ms_avg": 339600.71, "last_10_steps_tokens_per_second_per_gpu_avg": 771.918 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "5db2290dea4887a2ca9072c0d554707bf272945d", "command_topologies": [ { "gpus_per_node": "8", "nodes": "2", "recipe": "qwen35_vl_35b_a3b_pretrain_16gpu_h100_bf16_functional_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_pretrain_16gpu_h100_bf16_functional_config --dataset energon --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 50 --warmup_iters 10 dataset.path=work/data/datacomp/energon dataset.task_encoder.hf_processor_path=Qwen/Qwen3.6-35B-A3B dataset.task_encoder.hf_processor_revision=995ad96eacd98c81ed38be0c5b274b04031597b0 dataset.task_encoder.max_pixels=200704 dataset.do_validation=false dataset.pad_to_max_length=true scheduler.lr_decay_iters=50 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 checkpoint.load=null checkpoint.save=null validation.eval_iters=0 validation.eval_interval=0 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/h100-performance/resolved-config.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-pretrain-performance-h100", "expected_result": "The exact 16-H100 command completed all 50 DataComp Energon steps at TP1/PP2/CP1/EP8/ETP1, a 16/24 language-layer pipeline split, and MBS1/GBS512. The persisted config confirmed the 248,320-token model vocabulary, natural expert routing, HybridEP, selective core-attention, GDN output-norm, and MoE-activation recompute, scoped Transformer Engine CUDA graphs for language attention/router/preprocessing, a graph-free variable-shape vision encoder, and Transformer Engine fused cross entropy. Loss remained finite from 3.604611 to 3.458849 with zero skipped or NaN iterations. Steps 41-50 averaged 22,271.220 ms and 135.610 model TFLOP/s/GPU, and the process exited successfully without checkpoint output. The separate fixed-shape forced-balance tuning screen remains the source of the 185.150-TFLOP/s/GPU leader policy; it is not substituted for this real-data measurement.\n", "hardware": "H100", "last_verified": "2026-08-05", "metrics": { "final_loss": 3.458849, "initial_loss": 3.604611, "last_10_steps_model_tflops_per_gpu_avg": 135.61, "last_10_steps_step_time_ms_avg": 22271.22, "last_10_steps_tokens_per_second_per_gpu_avg": 5885.264 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.H100", "status": "verified", "variant": null, "workflow": "pretrain_performance" }, { "bridge_commit": "5db2290dea4887a2ca9072c0d554707bf272945d", "command_topologies": [ { "gpus_per_node": "4", "nodes": "2", "recipe": "qwen35_vl_35b_a3b_pretrain_8gpu_gb200_fp8mx_config", "total_gpus": 8 } ], "commands": [ "./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_pretrain_8gpu_gb200_fp8mx_config --max_steps 50 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-performance/resolved-config.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-6-35b-a3b-pretrain-performance-gb200", "expected_result": "On 8x GB200, the exact architecture-shared Qwen3.5/Qwen3.6-VL fixed-shape mock-data recipe completed 50 steps at TP1/PP1/CP1/EP8/ETP1, dense DP8 and expert DP1, MBS3/GBS480, and 20 microbatches per optimizer step. The persisted post-setup config confirmed forced expert balance, HybridEP, MXFP8, cuDNN LayerNorm, and Transformer Engine CUDA graphs for the MoE router and preprocessing scopes. Loss was finite from 12.81499 to 3.166235 with zero skipped or NaN iterations. Steps 41-50 averaged 26,880.420 ms and 210.360 model TFLOP/s/GPU, and the process exited successfully. This synthetic result is a throughput measurement, not a convergence claim.\n", "hardware": "GB200", "last_verified": "2026-07-30", "metrics": { "final_loss": 3.166235, "initial_loss": 12.81499, "last_10_steps_model_tflops_per_gpu_avg": 210.36, "last_10_steps_step_time_ms_avg": 26880.42, "last_10_steps_tokens_per_second_per_gpu_avg": 9142.714 }, "precision": "fp8_mx", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "Qwen/Qwen3.6-35B-A3B", "hf_revision": "995ad96eacd98c81ed38be0c5b274b04031597b0", "min_transformers_version": "5.8.1", "slug": "qwen3.6-35b-a3b", "source_card": "examples/model_verification_cards/qwen3.6-35b-a3b/card.yaml", "summary": "Performance scope: pretrain_performance.H100 uses the tuned 16-H100 library recipe with the same pinned DataComp source as functional pretraining. The corresponding real-data pretrain, SFT, and LoRA measurements are verified. pretrain_performance.GB200 retains its tuned fixed-shape synthetic benchmark. Functional timing and throughput remain sanity checks. This card verifies conversion, inference, and training.\n", "title": "qwen3_6_35b_a3b" }, { "architecture": "Qwen3_5ForConditionalGeneration", "base_container": "nvcr.io/nvidia/nemo:26.08.rc5", "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "entries": [ { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --gpus-per-node 1 --mem 512G --hf-model Qwen/Qwen3.8-27B --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 --megatron-path work/model-verification/qwen3.8-27b/cpu-megatron" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-hf-to-megatron-cpu", "expected_result": "The pinned-revision CPU import completed in 3m11s and created a reloadable iter_0000000 containing 8 nonempty files totaling 55,586,366,001 bytes. The converter selected Qwen35VLBridge, completed all 956 mappings, kept model weights on CPU, and reported 27,781,427,952 parameters. One shared runtime GB200 was used for Transformer Engine vision-module construction.\n", "hardware": null, "last_verified": "2026-08-14", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "tensor_parallel": "4", "total_gpus": 4 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --hf-model Qwen/Qwen3.8-27B --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 --megatron-path work/model-verification/qwen3.8-27b/gpu-megatron --tp 4" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-hf-to-megatron-gpu", "expected_result": "The pinned-revision four-GB200 import completed in 2m01s at TP4/PP1 and created an 11-file iter_0000000 totaling 55,592,033,255 bytes, including four distributed checkpoint shards. All 956 conversion mappings completed, and subsequent export and two independent inference runs reloaded it.\n", "hardware": null, "last_verified": "2026-08-14", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --gpus-per-node 1 --mem 512G --hf-model Qwen/Qwen3.8-27B --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 --megatron-path work/model-verification/qwen3.8-27b/cpu-megatron/iter_0000000 --hf-path work/model-verification/qwen3.8-27b/cpu-hf-export" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-megatron-to-hf-cpu", "expected_result": "CPU export completed in 2m16s, wrote every source tensor, and produced 26 files totaling 55,583,128,197 bytes. The index preserves the original 1,199-key set and exact 18-shard placement with 55,562,855,904 BF16 tensor bytes. Transformers strictly reloaded the export as Qwen3_5ForConditionalGeneration with empty missing, unexpected, mismatched, and error sets, then completed processor-native GB200 inference. Export normalizes vision_config.model_type from qwen3_5 to qwen3_5_vision and updates producer-version metadata without changing the architecture.\n", "hardware": null, "last_verified": "2026-08-14", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "tensor_parallel": "4", "total_gpus": 4 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 4 --hf-model Qwen/Qwen3.8-27B --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 --megatron-path work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --hf-path work/model-verification/qwen3.8-27b/gpu-hf-export --tp 4" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-megatron-to-hf-gpu", "expected_result": "The four-GB200 distributed export completed in 1m55s, wrote every source tensor, and produced 26 files totaling 55,583,128,222 bytes. Its index has the same 1,199 keys, 18-shard placement, and 55,562,855,904 BF16 tensor bytes as the pinned source and CPU export. Transformers strictly reloaded it with empty missing, unexpected, mismatched, and error sets and completed processor-native inference. Relative to CPU export, config.json only adds the metadata field vision_config.dtype=bfloat16.\n", "hardware": null, "last_verified": "2026-08-14", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "ab905ac72ccc9e45a081153655eda9ee469fdcff", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "tensor_parallel": "4", "total_gpus": 4 } ], "commands": [ "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 4 --task model-comparison --hf_model_path Qwen/Qwen3.8-27B --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 --model_class Qwen3_5ForConditionalGeneration --megatron_model_path work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --tp 4 --image_path work/data/medpix/verification.png --prompt \"What was observed in the imaging?\"" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-manual-forward-pass", "expected_result": "The pinned HF weights and imported TP4 checkpoint both produced 1-by-128-by-248320 logits from a 128-token input and a 256-by-1536 pixel tensor. Both selected token 248045, the assistant-start special token. Cosine similarity: 0.995276 (99.53%); the maximum and mean absolute logit differences were 1.125000 and 0.192757. This passes the required next-token match and 0.99 cosine-similarity gate. The input was MedPix-VQA revision 69eb219d477ab8699296da1fd6b89beb768042d3 validation row 0, case MPX1095; its 256-by-256 image has SHA-256 f5210fcff6dad525c80fcd0f462b274c9da4d634ccd4f1cc76578f1bfe13646d.\n", "hardware": null, "last_verified": "2026-08-14", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "tensor_parallel": "4", "total_gpus": 4 } ], "commands": [ "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 4 --task vlm-generation --hf_model_path Qwen/Qwen3.8-27B --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 --megatron_model_path work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --tp 4 --image_path work/data/medpix/verification.png --prompt \"What was observed in the imaging?\" --max_new_tokens 32" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-inference", "expected_result": "Two independent four-GB200 greedy-decoding runs loaded the imported TP4 checkpoint and consumed MedPix-VQA revision 69eb219d477ab8699296da1fd6b89beb768042d3 validation row 0, case MPX1095, using source-image SHA-256 f5210fcff6dad525c80fcd0f462b274c9da4d634ccd4f1cc76578f1bfe13646d. Both exited successfully with byte-identical 32-token completions and reached the 32-token maximum without EOS. The exact completion was: \"The user has provided an axial brain MRI image, likely T2-weighted given the bright CSF and the appearance of the structures. The image is from Med\".\n", "hardware": null, "last_verified": "2026-08-14", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "2e945d0734f2f0fb764320dda9c90139d02b792d", "command_topologies": [ { "gpus_per_node": "4", "nodes": "4", "recipe": "qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config --mode pretrain --deterministic --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 --save_dir work/model-verification/qwen3.8-27b/pretrain/checkpoints --save_interval 10 dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 checkpoint.load=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/pretrain/resolved-config.yaml scheduler.lr_decay_iters=20" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-pretrain-gb200", "expected_result": "The 16-GB200 MockVLM pretrain completed exactly 20 deterministic steps at TP4/PP4, sequence length 4096, and MBS2/GBS32. The language model and multimodal vision merger trained while the vision encoder remained frozen. Loss remained finite from 6.858159 to 3.955657 with finite gradients and zero skipped or NaN iterations. Steps 11-20 averaged 41,391.140 ms, 33.240 corrected model TFLOP/s/GPU, and 197.917 token slots/s/GPU. Checkpoints at iterations 10 and 20 each contain 20 files totaling 399,257,378,741 bytes, and the tracker records iteration 20. Both built-in 32-iteration validation and test passes completed at loss 3.907797. The process exited successfully in 26m31s.\n", "hardware": "GB200", "last_verified": "2026-08-20", "metrics": { "final_loss": 3.955657, "initial_loss": 6.858159, "last_10_steps_model_tflops_per_gpu_avg": 33.24, "last_10_steps_step_time_ms_avg": 41391.14, "last_10_steps_tokens_per_second_per_gpu_avg": 197.917 }, "precision": "bf16", "source_pointer": "items.pretrain.GB200", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "4", "nodes": "4", "recipe": "qwen35_vl_27b_sft_16gpu_h100_bf16_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft --dataset medpix --deterministic --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 --save_interval 50 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.pad_to_max_length=true tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 model.recompute_modules=null scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.8-27b/sft-recompute/checkpoints logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-recompute/resolved-config.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-sft-gb200", "expected_result": "The 16-GB200 run completed exactly 100 full-SFT MedPix steps at TP4/PP4, MBS4/GBS32, deterministic mode, and one-layer full-uniform activation recompute. Loss remained finite from 2.116404 to 1.429567 with zero skipped or NaN iterations. Steps 91-100 averaged 24,160.490 ms and 58.010 model TFLOP/s/GPU. Step-50 and step-100 saves each contain 20 files totaling 404,422,605,133 bytes, the tracker selects iteration 100, and the process exited successfully in 45m42s. The unchanged stock TP4/PP4 recipe without activation recompute reached the training loop but OOMed near the approximately 184.3-GiB per-GPU device limit; the successful command records the required config-only recompute overrides.\n", "hardware": "GB200", "last_verified": "2026-08-14", "metrics": { "final_loss": 1.429567, "initial_loss": 2.116404, "last_10_steps_model_tflops_per_gpu_avg": 58.01, "last_10_steps_step_time_ms_avg": 24160.49, "last_10_steps_tokens_per_second_per_gpu_avg": 339.066 }, "precision": "bf16", "source_pointer": "items.sft.GB200", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 }, { "gpus_per_node": "1", "nodes": "1", "total_gpus": 1 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --gpus-per-node 1 --mem 512G --hf-model Qwen/Qwen3.8-27B --hf-revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 --megatron-path work/model-verification/qwen3.8-27b/sft-recompute/checkpoints/iter_0000100 --hf-path work/model-verification/qwen3.8-27b/sft-recompute/hf-export", "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 1 --task hf-inference --hf-model work/model-verification/qwen3.8-27b/sft-recompute/hf-export --prompt \"What was observed in the imaging?\" --image work/data/medpix/verification.png --max-new-tokens 32 --chat-template --disable-thinking" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-sft-export-inference-gb200", "expected_result": "CPU export from the exact step-100 SFT checkpoint completed all 956 mappings in 2m39s and reported that every source tensor was written. The export contains 26 nonempty files totaling 55,583,128,197 bytes; its index maps 1,199 keys to 18 safetensors shards with 55,562,855,904 serialized BF16 tensor bytes. The unchanged strict HF verifier then reloaded the persisted export without any missing, unexpected, mismatched, or error entries on one GB200 and generated exactly 32 tokens at the 32-token maximum. The literal completion was: \"The imaging shows \u2022 CT: Hyperdense lesion in the right cerebellum.\\n\u2022 MRI: Well-circumscribed, homogenous\". Because deterministic mode is persisted in the training checkpoint, Transformer Engine requires nondeterministic algorithms to be explicitly disabled while constructing the Megatron model for export. An initial strict attempt without that environment setting failed before conversion and deleted its partial output. The successful retry explicitly disabled nondeterministic Transformer Engine algorithms through runtime wiring outside this public card and made no code change.\n", "hardware": "GB200", "last_verified": "2026-08-14", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB200", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "context_parallel": "2", "gpus_per_node": "4", "nodes": "4", "pipeline_parallel": "2", "recipe": "qwen35_vl_27b_sft_16gpu_h100_bf16_config", "sequence_length": "8192", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 --seq_length 8192 --pipeline_model_parallel_size 2 --context_parallel_size 2 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.enable_in_batch_packing=true dataset.defer_in_batch_packing_to_step=true dataset.in_batch_packing_pad_to_multiple_of=4 tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.calculate_per_token_loss=true model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 ddp.average_in_collective=false scheduler.lr_decay_iters=20 checkpoint.load=null checkpoint.save=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-long/resolved-config.yaml" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "sequence_packing": "in_batch" }, "entry_id": "qwen3-8-27b-sft-long-context-gb200", "expected_result": "The 16-GB200 run completed exactly 20 full-SFT steps at TP4/PP2/CP2, MBS4/GBS32, 8192 tokens, deferred processor-native in-batch packing with pad-to-multiple-of-4, per-token loss, and one-layer full-uniform activation recompute. Loss remained finite from 2.343622 to 1.378064 with zero skipped or NaN iterations. Steps 11-20 averaged 8,164.790 ms and 362.530 model TFLOP/s/GPU. The process exited successfully without checkpoint output. An earlier otherwise identical attempt with --deterministic reached the training loop but triggered the explicit guard that packed sequence does not support deterministic mode; the successful command omits that incompatible flag, as does the Qwen 3.5 MoE reference card.\n", "hardware": "GB200", "last_verified": "2026-08-14", "metrics": { "final_loss": 1.378064, "initial_loss": 2.343622, "last_10_steps_model_tflops_per_gpu_avg": 362.53, "last_10_steps_step_time_ms_avg": 8164.79, "last_10_steps_tokens_per_second_per_gpu_avg": 2006.665 }, "precision": "bf16", "source_pointer": "items.sft_long_context.GB200", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "4", "nodes": "1", "recipe": "qwen35_vl_27b_peft_2gpu_h100_bf16_config", "tensor_parallel": "4", "total_gpus": 4 } ], "commands": [ "./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen35_vl_27b_peft_2gpu_h100_bf16_config --mode lora --tensor_model_parallel_size 4 --dataset medpix --deterministic --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 --save_interval 50 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.pad_to_max_length=true tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 model.recompute_modules=null scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.8-27b/peft-tp4-recompute/checkpoints logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/peft-tp4-recompute/resolved-config.yaml" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-peft-gb200", "expected_result": "The four-GB200 run completed exactly 100 deterministic MedPix LoRA steps at TP4/PP1, MBS4/GBS32, and one-layer full-uniform activation recompute. Loss remained finite from 2.115661 to 1.558339 with zero skipped or NaN iterations. Steps 91-100 averaged 74,048.550 ms and 75.790 model TFLOP/s/GPU. Step-50 and step-100 adapter checkpoints each contain 8 files totaling 2,294,105,245 bytes, the tracker selects iteration 100, and the process exited successfully in 2h01m43s. The unchanged stock TP2 two-GPU recipe without activation recompute OOMed; a TP4 retry without recompute also OOMed near the approximately 184.3-GiB per-GPU device limit. The successful command records only the required topology and recompute config overrides, with no code change.\n", "hardware": "GB200", "last_verified": "2026-08-14", "metrics": { "final_loss": 1.558339, "initial_loss": 2.115661, "last_10_steps_model_tflops_per_gpu_avg": 75.79, "last_10_steps_step_time_ms_avg": 74048.55, "last_10_steps_tokens_per_second_per_gpu_avg": 442.52 }, "precision": "bf16", "source_pointer": "items.peft.GB200", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "d790e32a88654968021291d68c510f41d64df461", "command_topologies": [ { "gpus_per_node": "4", "nodes": "4", "recipe": "qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config --mode pretrain --deterministic --load_dir work/model-verification/qwen3.8-27b/pretrain/checkpoints --max_steps 20 --warmup_iters 2 --save_dir work/model-verification/qwen3.8-27b/pretrain-resume/checkpoints --save_interval 10 dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 checkpoint.ckpt_step=10 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/pretrain-resume/resolved-config.yaml scheduler.lr_decay_iters=20" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-checkpoint-resume-gb200", "expected_result": "A clean 16-GB200 attempt loaded the full iteration-10 checkpoint with optimizer, scheduler, and RNG state and directly executed steps 11-20. All losses and gradients were finite, the run wrote a complete iteration-20 checkpoint, and the first and final resumed losses stayed within the declared bound. However, steps 15-18 did not: the largest relative difference from the uninterrupted reference was 39.6205% at step 15. The resolved reference and resume configs differ only in the expected checkpoint source, step, and pretrained-checkpoint fallback. This card therefore makes no checkpoint-resume verification claim until the intermediate divergence is understood and a clean rerun matches the reference contract.\n", "hardware": "GB200", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.GB200", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "2e945d0734f2f0fb764320dda9c90139d02b792d", "command_topologies": [ { "gpus_per_node": "4", "nodes": "4", "recipe": "qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config", "total_gpus": 16 } ], "commands": [ "./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_pretrain_16gpu_gb200_bf16_mock_config --mode pretrain --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.8-27b/pretrain-performance/resolved-config.yaml scheduler.lr_decay_iters=20" ], "dimensions": {}, "enabled_features": {}, "entry_id": "qwen3-8-27b-pretrain-performance-gb200", "expected_result": "On 16 GB200 GPUs, the exact canonical BF16 language-and-projector MockVLM recipe completes 20 steps at TP2/PP1/CP1, DP8, sequence length 4096, and MBS2/GBS32. The language model and multimodal vision merger train while the vision encoder remains frozen. The recipe uses fused Gated DeltaNet execution without activation recompute and disables evaluation and checkpoint output for the bounded run. Loss remains finite from 6.852387 to 3.884668 with finite gradients and zero skipped or NaN iterations. Excluding first-iteration kernel compilation, steps 11-20 average 1,679.460 ms, 819.190 corrected model TFLOP/s/GPU, and 4,877.758 token slots/s/GPU; peak allocated memory is 174.120 GiB. The process exits successfully. This mock-data result is throughput evidence, not convergence evidence.\n", "hardware": "GB200", "last_verified": "2026-08-20", "metrics": { "final_loss": 3.884668, "initial_loss": 6.852387, "last_10_steps_model_tflops_per_gpu_avg": 819.19, "last_10_steps_step_time_ms_avg": 1679.46, "last_10_steps_tokens_per_second_per_gpu_avg": 4877.758, "peak_allocated_memory_gib": 174.12 }, "precision": "bf16", "source_pointer": "items.pretrain_performance.GB200", "status": "verified", "variant": null, "workflow": "pretrain_performance" } ], "hf_id": "Qwen/Qwen3.8-27B", "hf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "min_transformers_version": "5.8.0", "slug": "qwen3.8-27b", "source_card": "examples/model_verification_cards/qwen3.8-27b/card.yaml", "summary": "Performance scope: pretrain_performance.GB200 uses the tuned canonical language-and-projector MockVLM recipe and records 819.190 corrected model TFLOP/s/GPU on 16 GB200 GPUs; timing from all functional items remains sanity evidence. All workloads in this card target GB200 hardware. Conversion, deterministic inference, pretraining, SFT, long-context SFT, PEFT, and post-SFT export are verified. Full-language checkpoint resume remains unverified because four intermediate losses exceeded the comparison bound. No dense-27B Megatron FSDP recipe is available.\n", "title": "qwen3_8_27b" }, { "architecture": "Step3p5ForCausalLM", "base_container": "nvcr.io/nvidia/pytorch:26.04-py3", "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "entries": [ { "bridge_commit": "dd0fe603c598d1e031ada0f1af4c5040b237cdef", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model stepfun-ai/Step-3.5-Flash --hf-revision ab446a3de5e171ea341227e24bb1f090e1b771f7 --megatron-path work/model-verification/step35-flash/cpu-megatron --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-hf-to-megatron-cpu", "expected_result": "CPU import completed all mappings for the immutable 44-shard source, constructed 199,997,706,240 parameters, and created iter_0000000. The checkpoint reloaded successfully for CPU export; its subsequent strict audit covered all 804 source tensors.\n", "hardware": null, "last_verified": "2026-08-05", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "dd0fe603c598d1e031ada0f1af4c5040b237cdef", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model stepfun-ai/Step-3.5-Flash --hf-revision ab446a3de5e171ea341227e24bb1f090e1b771f7 --megatron-path work/model-verification/step35-flash/imported-megatron --torch-dtype bfloat16 --tp 1 --pp 1 --ep 8 --etp 1 --distributed-timeout-minutes 110 --trust-remote-code --low-memory-save" ], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-hf-to-megatron-gpu", "expected_result": "Distributed import completed at TP1/PP1/EP8/ETP1, constructed 33,525,780,480 parameters per rank, and created iter_0000000. The persisted checkpoint then reloaded successfully for both distributed export and checkpoint-backed generation.\n", "hardware": null, "last_verified": "2026-08-05", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model stepfun-ai/Step-3.5-Flash --hf-revision ab446a3de5e171ea341227e24bb1f090e1b771f7 --megatron-path work/model-verification/step35-flash/cpu-megatron/iter_0000000 --hf-path work/model-verification/step35-flash/cpu-hf-export --torch-dtype bfloat16 --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-megatron-to-hf-cpu", "expected_result": "CPU export reloaded the imported checkpoint and wrote 44 indexed BF16 shards with all 804 source keys, shapes, and dtypes. Of 398,768,626,944 serialized bytes, 395,600,878,848 bytes across 801 tensors matched the immutable source bitwise. The three published MTP output entries occupy 3,167,748,096 bytes and were intentionally materialized from Megatron's shared lm_head representation; each exported entry matched lm_head bitwise and used independent safetensors storage. A Transformers reload through the pinned checkpoint's metadata/RoPE compatibility view ignored only the three published inference-unused MTP layers, predicted the same next token as the source (' Paris'), and reached cosine similarity 0.997260 with maximum/mean absolute logit differences 1.320312/0.209649.\n", "hardware": null, "last_verified": "2026-08-05", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "0b5cc0206e77b4d18139c2ff4fc0d353ea260c20", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 1 --gpus-per-node 8 --hf-model stepfun-ai/Step-3.5-Flash --hf-revision ab446a3de5e171ea341227e24bb1f090e1b771f7 --megatron-path work/model-verification/step35-flash/imported-megatron/iter_0000000 --hf-path work/model-verification/step35-flash/hf-export --torch-dtype bfloat16 --tp 1 --pp 1 --ep 8 --etp 1 --distributed-timeout-minutes 110 --distributed-save --save-every-n-ranks 1 --no-progress --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-megatron-to-hf-gpu", "expected_result": "Distributed export reloaded the TP1/PP1/EP8/ETP1 checkpoint and wrote 44 indexed BF16 shards with all 804 source keys, shapes, and dtypes. The exact audit found 801 tensors and 395,600,878,848 bytes bit-identical to the source. Megatron-Core shares the output head with MTP, so the three source MTP output tensors were explicitly normalized to lm_head.weight; all three exported copies matched that tensor bitwise and had independent storage. The generic grouped-expert export path recovered from CUDA stack pressure by moving only the affected merge to CPU. A Transformers reload through the pinned checkpoint's metadata/RoPE compatibility view ignored only the three published inference-unused MTP layers, predicted the same next token as the source (' Paris'), and reached cosine similarity 0.997260 with maximum/mean absolute logit differences 1.320312/0.209649.\n", "hardware": null, "last_verified": "2026-08-05", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-manual-forward-pass", "expected_result": "The 2026-08-05 full-checkpoint attempt used the public model-comparison launcher and memory-bounded pinned-HF reference logits for input IDs [0, 671, 6102, 294, 8760, 344]. Hugging Face and Megatron both predicted token 11111 (' Paris'), but cosine similarity was 0.987008, below the required 0.99 gate; maximum and mean absolute logit differences were 4.937500 and 1.064947. Exact weight audits passed, so this remains an unverified runtime-architecture/configuration follow-up rather than a conversion-key mismatch. A future rerun must preserve the next-token match and reach cosine similarity of at least 0.99.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "dd0fe603c598d1e031ada0f1af4c5040b237cdef", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "1", "pipeline_parallel": "1", "tensor_parallel": "1", "total_gpus": 8 } ], "commands": [ "./scripts/inference/infer.sh --nodes 1 --gpus-per-node 8 --task legacy-full-prefix-generation --hf_model_path stepfun-ai/Step-3.5-Flash --hf-revision ab446a3de5e171ea341227e24bb1f090e1b771f7 --megatron_model_path work/model-verification/step35-flash/imported-megatron/iter_0000000 --tp 1 --pp 1 --ep 8 --etp 1 --prompt $'\\u003c|im_start|\\u003euser\\nReply with exactly OK and nothing else.\\u003c|im_end|\\u003e\\n\\u003c|im_start|\\u003eassistant' --max_new_tokens 32 --legacy-full-prefix --trust-remote-code" ], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-inference", "expected_result": "Two deterministic greedy runs reloaded the persisted TP1/PP1/EP8/ETP1 checkpoint and produced the byte-identical 7-token completion \"\\nOK\\n\\u003c/think\\u003e\\nOK\" before reaching EOS naturally, below the configured 32-token maximum. The load produced 6,464 known CPU-merge fallback warnings and briefly peaked at 80,974 MiB per rank, but cleanup reduced post-load residency to 65,466 MiB and the first forward peaked at 69,646 MiB. The original retained 77.6-GiB state, OOM, and empty generation did not recur.\n", "hardware": null, "last_verified": "2026-08-05", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": { "context_parallel_size": 8, "moe_dispatcher": "deepep" }, "entry_id": "step35-flash-pretrain-h100", "expected_result": "The public Step-3.5-Flash H100 recipe must complete a bounded 100-step run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-sft-h100", "expected_result": "A pinned-data 100-step full-SFT run must finish with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-sft-export-inference-h100", "expected_result": "A verified SFT checkpoint must export to Hugging Face, strictly reload, and produce deterministic checkpoint-backed inference. Training and post-SFT export verification are deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-sft-long-context-h100", "expected_result": "A dedicated packed long-context SFT run must complete 100 steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-peft-h100", "expected_result": "A Step-3.5-Flash PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, all five metrics, and a reloadable adapter checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "d5c416c50e60a6fa2ddb4a840d2ff036a08ce864", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "step35-flash-checkpoint-resume-h100", "expected_result": "A direct continuation must restore model, optimizer, scheduler, data-order, and RNG state, match declared sentinel losses, and save a reloadable checkpoint to a distinct output root. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "stepfun-ai/Step-3.5-Flash", "hf_revision": "ab446a3de5e171ea341227e24bb1f090e1b771f7", "min_transformers_version": "5.8.1", "slug": "step35-flash", "source_card": "examples/model_verification_cards/step35-flash/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. This card intentionally limits its current verification scope to model-level conversion, parity, and checkpoint-backed inference for the pinned Step-3.5-Flash checkpoint. Training verification is deferred and no training-support result is claimed by this card. Five model-level items are verified; manual HF/Megatron logit correlation remains unverified because the observed cosine similarity is below the required gate.\n", "title": "step35_flash" }, { "architecture": "GlmMoeDsaForCausalLM", "base_container": "nvcr.io/nvidia/nemo:26.08.rc3", "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "entries": [ { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-hf-to-megatron-cpu", "expected_result": "A CPU import of the pinned Hugging Face revision must complete every mapping and create a reloadable Megatron checkpoint. This workflow is deferred by this card.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "unverified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "2", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "2", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model zai-org/GLM-5 --hf-revision 4e6698ba8e85059d749020e3c4d2123719f23926 --megatron-path work/model-verification/glm5/gpu-megatron --torch-dtype bfloat16 --tp 1 --pp 2 --ep 8 --etp 2 --distributed-timeout-minutes 60 --low-memory-save" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-hf-to-megatron-gpu", "expected_result": "The pinned 32-H100 import exits successfully at TP1/PP2/EP8/ETP2, completes all 6,201 distributed mapping tasks, and persists a reloadable iter_0000000 checkpoint. Reload plus exact export projection covers all 59,079 tensors and 1,487,822,475,264 tensor-payload bytes in the 78-layer inference graph with matching keys, shapes, dtypes, and values. The 791 source tensors under model.layers.78 belong only to the intentionally disabled appended MTP auxiliary layer and are outside this item.\n", "hardware": null, "last_verified": "2026-08-10", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "c94d0bdb1c863f99c6227bb36fbc7f3ebc565243", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "2", "nodes": "4", "pipeline_parallel": "2", "tensor_parallel": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 4 --cpu-processes-per-node 8 --cpus-per-task 16 --mem 0 --exclusive --hf-model zai-org/GLM-5 --hf-revision 4e6698ba8e85059d749020e3c4d2123719f23926 --megatron-path work/model-verification/glm5/gpu-megatron/iter_0000000 --hf-path work/model-verification/glm5/cpu-hf-export --torch-dtype bfloat16 --tp 1 --pp 2 --ep 8 --etp 2 --distributed-timeout-minutes 240 --distributed-save --save-every-n-ranks 1 --no-progress" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-megatron-to-hf-cpu", "expected_result": "The 32-process distributed CPU export exits successfully and writes 280 safetensors shards. The exhaustive projection audit covers all 59,079 tensors and 1,487,822,475,264 tensor-payload bytes in the 78-layer inference graph with zero missing, unexpected, shape, dtype, or value mismatches. The 791 tensors under model.layers.78 belong only to the intentionally disabled appended MTP auxiliary layer and remain outside this item. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 1,629 state tensors, 743,911,199,232 parameters, and no loading discrepancies.\n", "hardware": null, "last_verified": "2026-08-26", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "7e2181dcc5d09a1a859dfb0c655c9a1cd486e0ac", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "2", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "2", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 4 --gpus-per-node 8 --hf-model zai-org/GLM-5 --hf-revision 4e6698ba8e85059d749020e3c4d2123719f23926 --megatron-path work/model-verification/glm5/gpu-megatron/iter_0000000 --hf-path work/model-verification/glm5/gpu-hf-export --torch-dtype bfloat16 --tp 1 --pp 2 --ep 8 --etp 2 --distributed-timeout-minutes 60 --distributed-save --save-every-n-ranks 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-megatron-to-hf-gpu", "expected_result": "The 32-H100 distributed export exits successfully after all 6,201 mapping tasks and writes 280 safetensors shards. Of those, 278 are byte-for-byte identical to the pinned source shards; the two shards that also contain excluded MTP keys match all 63 common tensors exactly. The composite audit covers all 59,079 inference-graph tensors and 1,487,822,475,264 tensor-payload bytes with zero key, shape, dtype, or value mismatch. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 1,629 state tensors, 743,911,199,232 parameters, and no loading discrepancies.\n", "hardware": null, "last_verified": "2026-08-10", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-manual-forward-pass", "expected_result": "A pinned first-token Hugging Face versus Megatron comparison must match the argmax token and meet the card's logit-correlation threshold. The full Hugging Face reference pass is deferred by this card.\n", "hardware": null, "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "unverified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "b58afee863f40b64304373830ae79f3f61a1af19", "command_topologies": [ { "expert_parallel": "32", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "8", "pipeline_parallel": "2", "tensor_parallel": "1", "total_gpus": 64 } ], "commands": [ "./scripts/inference/infer.sh --nodes 8 --gpus-per-node 8 --task legacy-full-prefix-generation --hf_model_path zai-org/GLM-5 --hf-revision 4e6698ba8e85059d749020e3c4d2123719f23926 --prompt \"What is artificial intelligence?\" --max_new_tokens 20 --legacy-full-prefix --tp 1 --pp 2 --ep 32 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-inference", "expected_result": "One deterministic BF16 run loads the pinned Hugging Face model directly\ninto Megatron across 64 H100 ranks at TP1/PP2/EP32/ETP1 and uses the\nflex HybridEP dispatcher. The inference graph intentionally excludes the\ncheckpoint's appended MTP auxiliary layer. It exits successfully after\nexactly 20 new greedy tokens. The literal completion, including its\nleading space, was:\n\" How is artificial intelligence going to change our lives?\"", "hardware": null, "last_verified": "2026-08-10", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-pretrain-h100", "expected_result": "A public GLM-5 H100 recipe must complete a bounded 100-step run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "unverified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-sft-h100", "expected_result": "A pinned-data 100-step full-SFT run must finish with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "unverified", "variant": null, "workflow": "sft" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-sft-export-inference-h100", "expected_result": "A verified SFT checkpoint must export to Hugging Face, strictly reload, and produce deterministic checkpoint-backed inference. Training and post-SFT export verification are deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-sft-long-context-h100", "expected_result": "A dedicated packed long-context SFT run must complete a bounded run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "unverified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-peft-h100", "expected_result": "A GLM-5 PEFT recipe with an audited adapter target set must complete a bounded run with finite loss, all five metrics, and a reloadable adapter checkpoint. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "unverified", "variant": null, "workflow": "peft" }, { "bridge_commit": "5db9c262411cf4561d44cdf76a134b423fdaacc2", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-checkpoint-resume-h100", "expected_result": "A direct continuation must restore model, optimizer, scheduler, data-order, and RNG state, match declared sentinel losses, and save a reloadable checkpoint to a distinct output root. Training is deferred by this card.\n", "hardware": "H100", "last_verified": null, "metrics": { "final_loss": null, "initial_loss": null, "last_10_steps_model_tflops_per_gpu_avg": null, "last_10_steps_step_time_ms_avg": null, "last_10_steps_tokens_per_second_per_gpu_avg": null }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "unverified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "zai-org/GLM-5", "hf_revision": "4e6698ba8e85059d749020e3c4d2123719f23926", "min_transformers_version": "5.12.1", "slug": "glm5", "source_card": "examples/model_verification_cards/glm5/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Model-level verification covers deterministic HybridEP inference, exact distributed GPU import/export, and exact distributed CPU export of the pinned GLM-5 Hugging Face checkpoint's 78-layer inference graph. The checkpoint's appended MTP auxiliary layer is intentionally disabled and excluded from those claims. CPU import, Hugging Face/Megatron forward parity, training, checkpoint resume, and post-SFT export/reload verification are deferred.\n", "title": "glm5" }, { "architecture": "GlmMoeDsaForCausalLM", "base_container": "nvcr.io/nvidia/pytorch:26.04-py3", "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "entries": [ { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device cpu --nodes 1 --hf-model zai-org/GLM-5.2 --hf-revision 4d67f66cc64d3219133b767c253b2ad1425c6c88 --megatron-path work/model-verification/glm5-2/cpu-megatron --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-hf-to-megatron-cpu", "expected_result": "Import exits successfully and creates a reloadable CPU checkpoint. Its strict round-trip comparison matches all 59,585 BF16 tensors and 753,329,940,480 elements bitwise against the immutable HF revision.\n", "hardware": null, "last_verified": "2026-07-22", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_cpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_cpu" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "13", "pipeline_parallel": "13", "tensor_parallel": "1", "total_gpus": 104 } ], "commands": [ "./scripts/conversion/convert.sh import --executor slurm --device gpu --nodes 13 --gpus-per-node 8 --hf-model zai-org/GLM-5.2 --hf-revision 4d67f66cc64d3219133b767c253b2ad1425c6c88 --megatron-path work/model-verification/glm5-2/gpu-megatron --torch-dtype bfloat16 --tp 1 --pp 13 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-hf-to-megatron-gpu", "expected_result": "Import exits successfully at TP1/PP13/EP8/ETP1 and creates a reloadable iter_0000000 checkpoint. Its strict round-trip comparison matches all 59,585 BF16 tensors and 753,329,940,480 elements bitwise against the immutable HF revision.\n", "hardware": null, "last_verified": "2026-07-22", "metrics": {}, "precision": "bf16", "source_pointer": "items.hf_to_megatron_gpu", "status": "verified", "variant": null, "workflow": "hf_to_megatron_gpu" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "nodes": "1" } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device cpu --nodes 1 --hf-model zai-org/GLM-5.2 --hf-revision 4d67f66cc64d3219133b767c253b2ad1425c6c88 --megatron-path work/model-verification/glm5-2/cpu-megatron --hf-path work/model-verification/glm5-2/cpu-hf-export --torch-dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-megatron-to-hf-cpu", "expected_result": "Export exits successfully; all 59,585 BF16 tensors and 753,329,940,480 elements match the immutable HF revision bitwise. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 743,377,000,704 parameters.\n", "hardware": null, "last_verified": "2026-07-22", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_cpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_cpu" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "13", "pipeline_parallel": "13", "tensor_parallel": "1", "total_gpus": 104 } ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 13 --gpus-per-node 8 --hf-model zai-org/GLM-5.2 --hf-revision 4d67f66cc64d3219133b767c253b2ad1425c6c88 --megatron-path work/model-verification/glm5-2/gpu-megatron --hf-path work/model-verification/glm5-2/gpu-hf-export-exact --torch-dtype bfloat16 --distributed-save --tp 1 --pp 13 --ep 8 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-megatron-to-hf-gpu", "expected_result": "Distributed export exits successfully; all 59,585 BF16 tensors and 753,329,940,480 elements match the immutable HF revision bitwise. Transformers strictly reloads the output as GlmMoeDsaForCausalLM with 743,377,000,704 parameters.\n", "hardware": null, "last_verified": "2026-07-22", "metrics": {}, "precision": "bf16", "source_pointer": "items.megatron_to_hf_gpu", "status": "verified", "variant": null, "workflow": "megatron_to_hf_gpu" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "expert_parallel": "4", "expert_tensor_parallel": "1", "pipeline_parallel": "13", "tensor_parallel": "1" } ], "commands": [ "uv run python -m torch.distributed.run --standalone --nproc_per_node=52 examples/conversion/compare_hf_and_megatron/compare.py --hf_model_path zai-org/GLM-5.2 --hf-revision 4d67f66cc64d3219133b767c253b2ad1425c6c88 --hf-logits-input work/model-verification/glm5-2/hf-forward-reference.safetensors --megatron-forward-mode standard --megatron_model_path work/model-verification/glm5-2/gpu-megatron --prompt \"The capital of France is\" --tp 1 --pp 13 --ep 4 --etp 1" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-manual-forward-pass", "expected_result": "The pinned HF and Megatron next-token results match at token 12089 (`Paris`). Cosine similarity is 0.999443, maximum absolute logit difference is 0.500000, and mean absolute logit difference is 0.063958.\n", "hardware": null, "last_verified": "2026-07-22", "metrics": {}, "precision": "bf16", "source_pointer": "items.manual_forward_pass", "status": "verified", "variant": null, "workflow": "manual_forward_pass" }, { "bridge_commit": "cd09c801474e5573ffceccf236b48b543c7a0b4a", "command_topologies": [ { "expert_parallel": "8", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "4", "pipeline_parallel": "4", "tensor_parallel": "1", "total_gpus": 32 } ], "commands": [ "./scripts/inference/infer.sh --nodes 4 --gpus-per-node 8 --task legacy-full-prefix-generation --hf_model_path zai-org/GLM-5.2 --hf-revision 4d67f66cc64d3219133b767c253b2ad1425c6c88 --prompt \"Write one concise sentence explaining why deterministic tests matter:\" --apply-chat-template --thinking-mode disabled --max_new_tokens 16 --legacy-full-prefix --tp 1 --pp 4 --ep 8 --etp 1 --pipeline-model-parallel-layout \"Etttttt|tttttttttttttttttttttttt|tttttttttttttttttttttttt|ttttttttttttttttttttttttL\"" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-inference", "expected_result": "One deterministic BF16 run imports the pinned complete HF model directly\ninto Megatron across 32 ranks at TP1/PP4/EP8/ETP1. It exits successfully\nafter exactly 16 new greedy tokens without constructing an MCore inference\ncontext. The legacy path recomputes the accumulated prefix at every step\nbecause cached inference is not yet supported for AbsorbedMLA. The literal\ncompletion was:\n\"1. **Analyze the Request:**\n * **Topic:**\"\nIt produced exactly 16 new tokens.", "hardware": null, "last_verified": "2026-08-02", "metrics": {}, "precision": "bf16", "source_pointer": "items.inference", "status": "verified", "variant": null, "workflow": "inference" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "gpus_per_node": "8", "nodes": "44", "pipeline_parallel": "11", "recipe": "glm52_pretrain_416gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 352 } ], "commands": [ "./scripts/training/train.sh --nodes 44 --gpus-per-node 8 --recipe glm52_pretrain_416gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --deterministic --seq_length 2048 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/wikitext103/wikitext103_glm52_text_document\"],null]' dataset.path_to_cache=work/cache/glm5-2/wikitext-pretrain dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 model.pipeline_model_parallel_size=11 model.virtual_pipeline_model_parallel_size=null model.microbatch_group_size_per_vp_stage=null model.pipeline_model_parallel_layout='Etttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttt|ttttmL' --save_dir work/model-verification/glm5-2/pretrain-reference/checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "deepep" }, "entry_id": "glm5-2-pretrain-h100", "expected_result": "The uninterrupted 352-H100 TP1/PP11/CP1/EP32/ETP1 run completed 100 real-data steps at GBS/MBS 1024/1 with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and flex/DeepEP dispatch. Loss remained finite from 13.15907 to 6.385243, final grad norm was 0.796, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 31,204.920 ms and 54.595 TFLOPS/GPU; peak observed GPU memory was 80,999 MiB. The matching step-50 and step-100 checkpoints contain model, optimizer, scheduler, data-order, and RNG state; the step-50 checkpoint reloads successfully.\n", "hardware": "H100", "last_verified": "2026-07-23", "metrics": { "final_loss": 6.385243, "initial_loss": 13.15907, "last_10_steps_model_tflops_per_gpu_avg": 54.595, "last_10_steps_step_time_ms_avg": 31204.92, "last_10_steps_tokens_per_second_per_gpu_avg": 190.926 }, "precision": "bf16", "source_pointer": "items.pretrain.H100", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "e79f756196e0b97ee3ad94263a896c120b32c1e5", "command_topologies": [ { "gpus_per_node": "4", "nodes": "48", "recipe": "glm52_pretrain_192gpu_gb200_bf16_config", "sequence_length": "4096", "total_gpus": 192 } ], "commands": [ "./scripts/training/train.sh --nodes 48 --gpus-per-node 4 --recipe glm52_pretrain_192gpu_gb200_bf16_config --mode pretrain --dataset megatron-indexed --deterministic --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01_text_document\"],null]' dataset.path_to_cache=work/cache/glm5-2/rp2-head-01-gb200 dataset.random_seed=1234 dataset.num_workers=8 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer/tokenizer.model tokenizer.use_tokenizer_vocab_size=false rng.seed=1234 scheduler.lr_decay_iters=100 model.moe_router_force_load_balancing=false checkpoint.load=null checkpoint.pretrained_checkpoint=null checkpoint.save_optim=true checkpoint.save_rng=true checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.finetune=false --save_dir work/model-verification/glm5-2/pretrain-rp2-gb200/checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-pretrain-gb200", "expected_result": "The 192-GB200 TP1/PP6/CP1/EP32/ETP1 run completed exactly 100 RP2 head_01 pretraining steps at GBS/MBS 1024/1 with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and all-to-all expert dispatch. The RP2 Llama SentencePiece tokenizer is used while retaining GLM-5.2's configured vocabulary size, so this is support verification rather than a cross-tokenizer convergence claim. Loss remained finite from 13.168150 to 6.523522, final grad norm was 2.775, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 64,786.540 ms and 102.520 TFLOPS/GPU. Step-50 and step-100 checkpoints contain model, optimizer, scheduler, data-order, and RNG state.\n", "hardware": "GB200", "last_verified": "2026-08-04", "metrics": { "final_loss": 6.523522, "initial_loss": 13.16815, "last_10_steps_model_tflops_per_gpu_avg": 102.52, "last_10_steps_step_time_ms_avg": 64786.54, "last_10_steps_tokens_per_second_per_gpu_avg": 337.189 }, "precision": "bf16", "source_pointer": "items.pretrain.GB200", "status": "verified", "variant": null, "workflow": "pretrain" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "context_parallel": "2", "gpus_per_node": "8", "nodes": "52", "recipe": "glm52_sft_416gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 416 } ], "commands": [ "./scripts/training/train.sh --nodes 52 --gpus-per-node 8 --recipe glm52_sft_416gpu_h100_bf16_config --mode sft --pretrained_checkpoint work/model-verification/glm5-2/gpu-megatron/iter_0000000 --max_steps 100 --seq_length 2048 dataset.hf_output_root=work/data/glm5-2/tulu3-full-sft-pad32 model.context_parallel_size=2 model.virtual_pipeline_model_parallel_size=null model.pipeline_model_parallel_layout='Etttttt|tttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttt|tttt|tttt|tttt|ttttmL' --save_dir work/model-verification/glm5-2/sft-functional/checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 2, "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "glm5-2-sft-h100", "expected_result": "The immutable-revision 416-H100 run completes exactly 100 full-SFT steps at TP1/PP13/CP2/EP32/ETP1, GBS/MBS 32/1, with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and natural routing. Pad-32 packing is 99.60% efficient and contains 4,097,215 supervised tokens. Loss remains finite from 1.909875 to 0.6634203 with zero skipped or NaN iterations; the final grad norm is 0.895 and the step-100 checkpoint reloads for export.\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": { "final_loss": 0.6634203, "initial_loss": 1.909875, "last_10_steps_model_tflops_per_gpu_avg": 10.154, "last_10_steps_step_time_ms_avg": 4436.29, "last_10_steps_tokens_per_second_per_gpu_avg": 35.511 }, "precision": "bf16", "source_pointer": "items.sft.H100", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "cc190939aad0e1ee0edfb98c5455ae3fe833140b", "command_topologies": [ { "gpus_per_node": "4", "nodes": "48", "recipe": "glm52_sft_192gpu_gb200_bf16_config", "total_gpus": 192 } ], "commands": [ "./scripts/training/train.sh --nodes 48 --gpus-per-node 4 --recipe glm52_sft_192gpu_gb200_bf16_config --mode sft --pretrained_checkpoint work/model-verification/glm5-2/hf-4d67f66cc64d3219133b767c253b2ad1425c6c88 --max_steps 100 tokenizer.chat_template_path=work/templates/glm52_chat_template_generation.jinja --save_dir work/model-verification/glm5-2/sft-gb200-cudnn/checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 4, "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "glm5-2-sft-gb200", "expected_result": "The immutable-revision 192-GB200 run completed exactly 100 full-SFT steps at TP1/PP6/CP4/EP32/ETP1, DP8, and GBS/MBS 8/1 without gradient accumulation. It used the pinned Tulu 3 train[:10000] split with the GLM assistant-only chat template, 8,192-token offline packs, pad-8 sequence alignment, an 8,184-token single-sequence cap, constant-shape cu_seqlens, cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, natural routing, and HybridEP. Packing is 99.72% efficient across 421 rows with a 23.75 packing factor. The 100-step window contains 6,553,600 token slots and 4,283,561 supervised tokens. Metadata and Parquet SHA-256 values are cbd964c686b74f82804ceea2df75bb00fbe28648f0458a641b9db27977a55b1b and b2c6f71a6b5add331750956622073cc8ced0bb6b2742e1d7a0e1a9f763cbaa0c. Loss remained finite from 1.704979 to 0.3682597, final grad norm was 0.873, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 8,593.990 ms and 12.446 TFLOPS/GPU. A complete step-100 distributed checkpoint was written for the future GB200 export/reload gate.\n", "hardware": "GB200", "last_verified": "2026-08-04", "metrics": { "final_loss": 0.3682597, "initial_loss": 1.704979, "last_10_steps_model_tflops_per_gpu_avg": 12.446, "last_10_steps_step_time_ms_avg": 8593.99, "last_10_steps_tokens_per_second_per_gpu_avg": 39.718 }, "precision": "bf16", "source_pointer": "items.sft.GB200", "status": "verified", "variant": null, "workflow": "sft" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "expert_parallel": "32", "expert_tensor_parallel": "1", "gpus_per_node": "8", "nodes": "52", "pipeline_parallel": "13", "tensor_parallel": "1", "total_gpus": 416 }, {} ], "commands": [ "./scripts/conversion/convert.sh export --executor slurm --device gpu --nodes 52 --gpus-per-node 8 --hf-model zai-org/GLM-5.2 --hf-revision 4d67f66cc64d3219133b767c253b2ad1425c6c88 --megatron-path work/model-verification/glm5-2/sft-functional/checkpoints/iter_0000100 --hf-path work/model-verification/glm5-2/sft-hf-export --torch-dtype bfloat16 --distributed-save --tp 1 --pp 13 --ep 32 --etp 1", "uv run python skills/create-model-verification-card/scripts/verify_hf_inference.py --hf-model work/model-verification/glm5-2/sft-hf-export --prompt \"Write one concise sentence explaining why deterministic tests matter:\" --max-new-tokens 8 --chat-template --disable-thinking --device cpu --dtype bfloat16" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-sft-export-inference-h100", "expected_result": "The step-100 full-SFT checkpoint exports successfully and strictly reloads with Transformers. One greedy run produces exactly 8 tokens with the completion \"Deterministic tests matter because they provide consistent\".\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.H100", "status": "verified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [], "commands": [], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-sft-export-inference-gb200", "expected_result": "Export and deterministic HF reload/inference from the GLM-5.2 SFT checkpoint have not been run for this card.\n", "hardware": "GB200", "last_verified": null, "metrics": {}, "precision": "bf16", "source_pointer": "items.sft_export_inference.GB200", "status": "unverified", "variant": null, "workflow": "sft_export_inference" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "gpus_per_node": "8", "nodes": "76", "recipe": "glm52_sft_608gpu_h100_bf16_200k_config", "sequence_length": "200000", "total_gpus": 608 } ], "commands": [ "./scripts/training/train.sh --nodes 76 --gpus-per-node 8 --recipe glm52_sft_608gpu_h100_bf16_200k_config --mode sft --pretrained_checkpoint work/model-verification/glm5-2/gpu-megatron/iter_0000000 --max_steps 20 --seq_length 200000 dataset.dataset_root=work/data/glm5-2/synthetic-200k-260 train.empty_unused_memory_level=2 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 32, "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "glm5-2-sft-long-context-h100", "expected_result": "The 608-H100 TP1/PP19/CP32/EP32/ETP1 run completed exactly 20 optimizer steps over 200,000-token packed rows with cuDNN DSA, sparse indexer loss coefficient 0.001, and MTP1. Loss remained finite from 8.283127 to 2.557671, final grad norm was 10838.703, and every step recorded zero skipped and NaN iterations. The last 10 steps averaged 36,603.370 ms and 39.648 TFLOPS/GPU; peak observed GPU memory was 81,005 MiB.\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": { "final_loss": 2.557671, "initial_loss": 8.283127, "last_10_steps_model_tflops_per_gpu_avg": 39.648, "last_10_steps_step_time_ms_avg": 36603.37, "last_10_steps_tokens_per_second_per_gpu_avg": 116.828 }, "precision": "bf16", "source_pointer": "items.sft_long_context.H100", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "bdb637740154729a2409645703a0b3c78c38fa8c", "command_topologies": [ { "gpus_per_node": "4", "nodes": "48", "recipe": "glm52_sft_192gpu_gb200_bf16_128k_config", "total_gpus": 192 } ], "commands": [ "./scripts/training/train.sh --nodes 48 --gpus-per-node 4 --recipe glm52_sft_192gpu_gb200_bf16_128k_config --mode sft --pretrained_checkpoint work/model-verification/glm5-2/hf-4d67f66cc64d3219133b767c253b2ad1425c6c88 --max_steps 20 tokenizer.chat_template_path=work/templates/glm52_chat_template_generation.jinja dataset.dataset_root=work/data/glm5-2/synthetic-long-sft-128k dataset.max_train_samples=1120 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "context_parallel_size": 32, "moe_dispatcher": "hybridep", "sequence_packing": "offline" }, "entry_id": "glm5-2-sft-long-context-gb200", "expected_result": "The immutable-revision 192-GB200 run completed exactly 20 optimizer steps at TP1/PP6/CP32/EP32/ETP1, DP1, and GBS/MBS 56/1 with cuDNN DSA, MTP1, HybridEP, and full activation recompute. The explicit PP6 decoder layout [14, 16, 12, 12, 12, 12] completed without OOM using 32 single-sequence, 131,072-token synthetic packs. Loss remained finite from 0.4159648 to 0.000511673 with zero skipped or NaN iterations across 146,786,080 supervised tokens. The final 10 steps averaged 143,248.820 ms and 88.848 TFLOPS/GPU. This verifies long-context execution only; the 8K pinned-Tulu run provides real-data convergence evidence.\n", "hardware": "GB200", "last_verified": "2026-08-05", "metrics": { "final_loss": 0.000511673, "initial_loss": 0.4159648, "last_10_steps_model_tflops_per_gpu_avg": 88.848, "last_10_steps_step_time_ms_avg": 143248.82, "last_10_steps_tokens_per_second_per_gpu_avg": 266.874 }, "precision": "bf16", "source_pointer": "items.sft_long_context.GB200", "status": "verified", "variant": null, "workflow": "sft_long_context" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "gpus_per_node": "8", "nodes": "26", "recipe": "glm52_peft_208gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 208 } ], "commands": [ "./scripts/training/train.sh --nodes 26 --gpus-per-node 8 --recipe glm52_peft_208gpu_h100_bf16_config --mode lora --pretrained_checkpoint work/model-verification/glm5-2/gpu-megatron/iter_0000000 --max_steps 100 --seq_length 2048 dataset.hf_output_root=work/data/glm5-2/tulu3-peft-pad4 --save_dir work/model-verification/glm5-2/peft-lora/checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "moe_dispatcher": "deepep", "sequence_packing": "offline" }, "entry_id": "glm5-2-peft-h100", "expected_result": "The immutable-revision 208-H100 run completes exactly 100 LoRA steps at TP1/PP13/CP1/EP16/ETP1, GBS/MBS 32/1, with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and natural routing. Pad-4 packing is 99.45% efficient and contains 4,251,129 supervised tokens. Loss remains finite from 1.909421 to 0.8835775 with zero skipped or NaN iterations. The final grad norm is 0.101; the checkpoint contains exactly 790 BF16 rank-8 adapter tensors and no base-model weights.\n", "hardware": "H100", "last_verified": "2026-07-22", "metrics": { "final_loss": 0.8835775, "initial_loss": 1.909421, "last_10_steps_model_tflops_per_gpu_avg": 19.423, "last_10_steps_step_time_ms_avg": 4638.55, "last_10_steps_tokens_per_second_per_gpu_avg": 67.926 }, "precision": "bf16", "source_pointer": "items.peft.H100", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "gpus_per_node": "4", "nodes": "48", "recipe": "glm52_peft_192gpu_gb200_bf16_config", "total_gpus": 192 } ], "commands": [ "./scripts/training/train.sh --nodes 48 --gpus-per-node 4 --recipe glm52_peft_192gpu_gb200_bf16_config --mode lora --pretrained_checkpoint work/model-verification/glm5-2/gpu-megatron/iter_0000000 --max_steps 100 tokenizer.chat_template_path=work/templates/glm52_chat_template_generation.jinja --save_dir work/model-verification/glm5-2/peft-gb200-cudnn/checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": { "sequence_packing": "offline" }, "entry_id": "glm5-2-peft-gb200", "expected_result": "The immutable-revision 192-GB200 run completed exactly 100 LoRA steps at TP1/PP6/CP1/EP32/ETP1, GBS/MBS 32/1, with offline-packed Tulu 3 data, cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and all-to-all expert dispatch. Loss remained finite from 1.831100 to 0.8957523, final grad norm was 0.101, and every step recorded zero skipped and NaN iterations. The step-100 adapter checkpoint was written for reload validation.\n", "hardware": "GB200", "last_verified": "2026-07-24", "metrics": { "final_loss": 0.8957523, "initial_loss": 1.8311, "last_10_steps_model_tflops_per_gpu_avg": 1.423, "last_10_steps_step_time_ms_avg": 68588.27, "last_10_steps_tokens_per_second_per_gpu_avg": 4.977 }, "precision": "bf16", "source_pointer": "items.peft.GB200", "status": "verified", "variant": null, "workflow": "peft" }, { "bridge_commit": "8645cda8738cbb87b8b998ad2d09341eef2510c7", "command_topologies": [ { "gpus_per_node": "8", "nodes": "44", "pipeline_parallel": "11", "recipe": "glm52_pretrain_416gpu_h100_bf16_config", "sequence_length": "2048", "total_gpus": 352 } ], "commands": [ "./scripts/training/train.sh --nodes 44 --gpus-per-node 8 --recipe glm52_pretrain_416gpu_h100_bf16_config --mode pretrain --dataset megatron-indexed --deterministic --seq_length 2048 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/wikitext103/wikitext103_glm52_text_document\"],null]' dataset.path_to_cache=work/cache/glm5-2/wikitext-pretrain dataset.random_seed=1234 rng.seed=1234 scheduler.lr_decay_iters=100 model.pipeline_model_parallel_size=11 model.virtual_pipeline_model_parallel_size=null model.microbatch_group_size_per_vp_stage=null model.pipeline_model_parallel_layout='Etttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttttttt|tttt|ttttmL' --load_dir work/model-verification/glm5-2/pretrain-reference/checkpoints --save_dir work/model-verification/glm5-2/pretrain-resume/checkpoints --save_interval 0 checkpoint.ckpt_step=50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-checkpoint-resume-h100", "expected_result": "The run reloads the corrected reference's step-50 model, optimizer, scheduler, data-order, and RNG state and completes steps 51-100 with zero skipped or NaN iterations. Step 51 matches exactly at 7.219822. Step 100 is 6.372624 versus the uninterrupted reference's 6.385243, an absolute difference of 0.012619 within the allowed 0.063853. Final grad norm is 2.169; the last 10 steps average 32,002.290 ms and 53.235 TFLOPS/GPU. Peak observed allocator memory is 65.230 GiB allocated and 69.684 GiB reserved.\n", "hardware": "H100", "last_verified": "2026-07-23", "metrics": { "final_loss": 6.372624, "initial_loss": 7.219822, "last_10_steps_model_tflops_per_gpu_avg": 53.235, "last_10_steps_step_time_ms_avg": 32002.29, "last_10_steps_tokens_per_second_per_gpu_avg": 186.168 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.H100", "status": "verified", "variant": null, "workflow": "checkpoint_resume" }, { "bridge_commit": "e79f756196e0b97ee3ad94263a896c120b32c1e5", "command_topologies": [ { "gpus_per_node": "4", "nodes": "48", "recipe": "glm52_pretrain_192gpu_gb200_bf16_config", "sequence_length": "4096", "total_gpus": 192 } ], "commands": [ "./scripts/training/train.sh --nodes 48 --gpus-per-node 4 --recipe glm52_pretrain_192gpu_gb200_bf16_config --mode pretrain --dataset megatron-indexed --deterministic --seq_length 4096 --max_steps 100 --lr 3e-4 --min_lr 3e-5 --warmup_iters 40 'dataset.blend=[[\"work/data/rp2/head_01_text_document\"],null]' dataset.path_to_cache=work/cache/glm5-2/rp2-head-01-gb200 dataset.random_seed=1234 dataset.num_workers=8 tokenizer.tokenizer_type=SentencePieceTokenizer tokenizer.tokenizer_model=work/data/rp2/tokenizer/tokenizer.model tokenizer.use_tokenizer_vocab_size=false rng.seed=1234 scheduler.lr_decay_iters=100 model.moe_router_force_load_balancing=false checkpoint.ckpt_step=50 checkpoint.save_optim=true checkpoint.save_rng=true checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.finetune=false --load_dir work/model-verification/glm5-2/pretrain-rp2-gb200/checkpoints --save_dir work/model-verification/glm5-2/pretrain-rp2-resume-gb200/checkpoints --save_interval 0 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null" ], "dimensions": {}, "enabled_features": {}, "entry_id": "glm5-2-checkpoint-resume-gb200", "expected_result": "The run reloads the RP2 reference's step-50 model, optimizer, scheduler, data-order, and RNG state and completes steps 51-100 with zero skipped or NaN iterations. Step 51 is 7.233636 versus 7.233639 in the uninterrupted reference, a relative difference of 4.1e-7. Step 100 is 6.503141 versus 6.523522, a relative difference of 0.003124; both satisfy the 1e-6 absolute plus 1% relative tolerance. Final grad norm is 1.690; the last 10 steps average 60,079.390 ms and 110.552 TFLOPS/GPU.\n", "hardware": "GB200", "last_verified": "2026-08-04", "metrics": { "final_loss": 6.503141, "initial_loss": 7.233636, "last_10_steps_model_tflops_per_gpu_avg": 110.552, "last_10_steps_step_time_ms_avg": 60079.39, "last_10_steps_tokens_per_second_per_gpu_avg": 363.608 }, "precision": "bf16", "source_pointer": "items.checkpoint_resume.GB200", "status": "verified", "variant": null, "workflow": "checkpoint_resume" } ], "hf_id": "zai-org/GLM-5.2", "hf_revision": "4d67f66cc64d3219133b767c253b2ad1425c6c88", "min_transformers_version": "5.12.1", "slug": "glm5-2", "source_card": "examples/model_verification_cards/glm5-2/card.yaml", "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. GLM-5.2 H100 support verification currently covers exact GPU checkpoint round-trip on CPU and GPU, HF-to-Megatron forward equivalence, bounded full SFT with export/reload inference, 200K-context CP32 SFT, bounded PEFT, and bounded pretraining with a reloadable middle checkpoint and verified continuation. GB200 support verification currently covers bounded pretraining, bounded full SFT, 128K packed long-context SFT, bounded PEFT, bounded checkpoint resume, and deterministic legacy Megatron inference; post-SFT export/reload inference remains pending.\n", "title": "glm5_2" } ], "schema_version": 1, "source": "examples/model_verification_cards/*/card.yaml" }