Megatron Core
Table of Contents
About Megatron Core
Get Started
Basic Usage
Supported Models
Advanced Features
Developer Guide
API Reference
core
core.models
core.models.T5
core.models.mimo
core.models.common
core.models.audio
core.models.huggingface
core.models.multimodal
core.models.vision
core.models.gpt
core.models.hybrid
core.models.mamba
core.models.bert
core.models.backends
core.datasets
core.datasets.gpt_dataset
core.datasets.blended_megatron_dataset_builder
core.datasets.helpers
core.datasets.megatron_dataset
core.datasets.t5_dataset
core.datasets.multimodal_dataset
core.datasets.blended_megatron_dataset_config
core.datasets.bert_dataset
core.datasets.indexed_dataset
core.datasets.masked_dataset
core.datasets.data_schedule
core.datasets.blended_dataset
core.datasets.utils_s3
core.datasets.object_storage_utils
core.datasets.utils
core.ssm
core.ssm.ops
core.ssm.gated_delta_net
core.ssm.mamba_mixer
core.ssm.causal_conv1d
core.ssm.mamba_hybrid_layer_allocation
core.ssm.mlp_layer
core.ssm.mamba_block
core.ssm.triton_cache_manager
core.ssm.mamba_layer
core.ssm.mamba_context_parallel
core.ssm.utils
core.post_training
core.post_training.modelopt
core.optimizer
core.optimizer.cpu_offloading
core.optimizer.optimizer_config
core.optimizer.qk_clip
core.optimizer.grad_scaler
core.optimizer.emerging_optimizers
core.optimizer.optimizer
core.optimizer.layer_wise_optimizer
core.optimizer.distrib_optimizer
core.optimizer.param_layout
core.optimizer.optimizer_cuda_graph
core.optimizer.fully_sharded_optimizer
core.optimizer.muon
core.optimizer.clip_grads
core.transformer
core.transformer.custom_layers
core.transformer.moe
core.transformer.cuda_graph_config
core.transformer.spec_utils
core.transformer.transformer_block
core.transformer.dot_product_attention
core.transformer.transformer_config
core.transformer.mlp
core.transformer.torch_layer_norm
core.transformer.module
core.transformer.attention
core.transformer.cuda_graphs
core.transformer.multi_latent_attention
core.transformer.multi_token_prediction
core.transformer.enums
core.transformer.torch_norm
core.transformer.identity_op
core.transformer.transformer_layer
core.transformer.fsdp_dtensor_checkpoint
core.transformer.mla_qk_norm_config
core.transformer.pipeline_parallel_layer_layout
core.transformer.utils
core.tokenizers
core.tokenizers.text
core.tokenizers.vision
core.tokenizers.base_tokenizer
core.tokenizers.megatron_tokenizer
core.export
core.export.trtllm
core.export.export_config
core.export.model_type
core.export.data_type
core.resharding
core.resharding.nvshmem_copy_service
core.resharding.copy_services
core.resharding.transforms
core.resharding.planner
core.resharding.refit
core.resharding.execution
core.resharding.utils
core.quantization
core.quantization.quant_config
core.quantization.utils
core.pipeline_parallel
core.pipeline_parallel.combined_1f1b
core.pipeline_parallel.fine_grained_activation_offload
core.pipeline_parallel.bridge_communicator
core.pipeline_parallel.hybrid_cp_schedule
core.pipeline_parallel.p2p_communication
core.pipeline_parallel.multimodule_communicator
core.pipeline_parallel.schedules
core.pipeline_parallel.utils
core.fusions
core.fusions.fused_bias_swiglu
core.fusions.fused_bias_geglu
core.fusions.fused_cross_entropy
core.fusions.fused_indices_converter
core.fusions.fused_bias_dropout
core.fusions.fused_softmax
core.fusions.fused_bias_gelu
core.fusions.fused_weighted_squared_relu
core.fusions.fused_layer_norm
core.fusions.fused_mla_yarn_rope_apply
core.fusions.fused_pad_routing_map
core.tensor_parallel
core.tensor_parallel.gtp_api
core.tensor_parallel.data
core.tensor_parallel.layers
core.tensor_parallel.inference_layers
core.tensor_parallel.cross_entropy
core.tensor_parallel.gtp_cuda_graphs
core.tensor_parallel.mappings
core.tensor_parallel.generalized_tensor_parallelism
core.tensor_parallel.random
core.tensor_parallel.utils
core.extensions
core.extensions.transformer_engine
core.extensions.transformer_engine_spec_provider
core.extensions.kitchen
core.inference
core.inference.sampling
core.inference.contexts
core.inference.data_parallel_inference_coordinator
core.inference.quantization
core.inference.moe
core.inference.engines
core.inference.apis
core.inference.disaggregation
core.inference.text_generation_server
core.inference.text_generation_controllers
core.inference.model_inference_wrappers
core.inference.scheduler
core.inference.unified_memory
core.inference.batch_dimensions_utils
core.inference.shards_spec
core.inference.inference_request
core.inference.config
core.inference.async_stream
core.inference.communication_utils
core.inference.sampling_params
core.inference.symmetric_memory
core.inference.headers
core.inference.inference_client
core.inference.shards
core.inference.common_inference_params
core.inference.utils
core.distributed
core.distributed.fsdp
core.distributed.finalize_model_grads
core.distributed.param_and_grad_buffer
core.distributed.torch_fully_sharded_data_parallel_config
core.distributed.data_parallel_base
core.distributed.distributed_data_parallel_config
core.distributed.distributed_data_parallel
core.distributed.torch_fully_sharded_data_parallel
core.distributed.reduce_scatter_with_fp32_accumulation
core.dist_checkpointing
core.dist_checkpointing.strategies
core.dist_checkpointing.tensor_aware_state_dict
core.dist_checkpointing.mapping
core.dist_checkpointing.gpt_checkpoint_interop
core.dist_checkpointing.state_dict_utils
core.dist_checkpointing.exchange_utils
core.dist_checkpointing.optimizer
core.dist_checkpointing.core
core.dist_checkpointing.validation
core.dist_checkpointing.dict_utils
core.dist_checkpointing.serialization
core.dist_checkpointing.utils
core.typed_torch
core.energy_monitor
core.nccl_allocator
core.packed_seq_params
core.fault_injector
core.safe_globals
core.inference_params
core.config
core._slurm_utils
core.msc_utils
core.optimizer_param_scheduler
core.hyper_comm_grid
core.rerun_state_machine
core.full_cuda_graph
core.fp8_utils
core.fp4_utils
core.enums
core._rank_utils
core.jit
core.process_groups_config
core.model_parallel_config
core.recompute
core.timers
core.activations
core.parallel_state
core.utils
core.num_microbatches_calculator
core.config_logger
core.package_info
Resources
core.models.common.language_module
core.models.common.language_module.language_module
