core.inference

Subpackages

• core.inference.text_generation_controllers
  • core.inference.text_generation_controllers.encoder_decoder_text_generation_controller
  • core.inference.text_generation_controllers.mtp_utils_pytorch
  • core.inference.text_generation_controllers.mtp_utils_triton
  • core.inference.text_generation_controllers.text_generation_controller
  • core.inference.text_generation_controllers.vlm_text_generation_controller
• core.inference.contexts
  • core.inference.contexts.mamba_slot_allocator
  • core.inference.contexts.kv_block_allocator
  • core.inference.contexts.dynamic_context
  • core.inference.contexts.routing_metadata
  • core.inference.contexts.gpu_view
  • core.inference.contexts.fused_kv_append_kernel
  • core.inference.contexts.base_context
  • core.inference.contexts.static_context
• core.inference.engines
  • core.inference.engines.mcore_engine
  • core.inference.engines.dynamic_engine
  • core.inference.engines.async_zmq_communicator
  • core.inference.engines.abstract_engine
  • core.inference.engines.static_engine
• core.inference.moe
  • core.inference.moe.fused_moe
  • core.inference.moe.permute
  • core.inference.moe.activations
  • core.inference.moe.vllm_fused_moe
  • core.inference.moe.metadata
• core.inference.sampling
  • core.inference.sampling.flashinfer_sampling
  • core.inference.sampling.torch_sampling
  • core.inference.sampling.base
• core.inference.text_generation_server
  • core.inference.text_generation_server.dynamic_text_gen_server
    • core.inference.text_generation_server.dynamic_text_gen_server.endpoints
    • core.inference.text_generation_server.dynamic_text_gen_server.text_generation_server
    • core.inference.text_generation_server.dynamic_text_gen_server.tokenization
  • core.inference.text_generation_server.run_mcore_engine
  • core.inference.text_generation_server.text_generation_server
  • core.inference.text_generation_server.tokenization
• core.inference.model_inference_wrappers
  • core.inference.model_inference_wrappers.gpt
    • core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper
  • core.inference.model_inference_wrappers.t5
    • core.inference.model_inference_wrappers.t5.t5_inference_wrapper
  • core.inference.model_inference_wrappers.abstract_model_inference_wrapper
• core.inference.quantization
  • core.inference.quantization.utils
  • core.inference.quantization.mxfp8_quantize
  • core.inference.quantization.mxfp8_tensor
• core.inference.apis
  • core.inference.apis.serve_config
  • core.inference.apis._llm_base
  • core.inference.apis.llm
  • core.inference.apis.async_llm

Submodules

• core.inference.symmetric_memory
• core.inference.communication_utils
• core.inference.async_stream
• core.inference.config
• core.inference.unified_memory
• core.inference.utils
• core.inference.batch_dimensions_utils
• core.inference.inference_client
• core.inference.sampling_params
• core.inference.common_inference_params
• core.inference.scheduler
• core.inference.data_parallel_inference_coordinator
• core.inference.headers
• core.inference.inference_request