core.inference

Subpackages

• core.inference.sampling
  • core.inference.sampling.torch_sampling
  • core.inference.sampling.flashinfer_sampling
  • core.inference.sampling.base
• core.inference.contexts
  • core.inference.contexts.routing_metadata
  • core.inference.contexts.fused_kv_append_kernel
  • core.inference.contexts.mamba_slot_allocator
  • core.inference.contexts.gpu_view
  • core.inference.contexts.base_context
  • core.inference.contexts.static_context
  • core.inference.contexts.kv_block_allocator
  • core.inference.contexts.dynamic_context
• core.inference.data_parallel_inference_coordinator
  • core.inference.data_parallel_inference_coordinator.state
  • core.inference.data_parallel_inference_coordinator.handlers
  • core.inference.data_parallel_inference_coordinator.coordinator
• core.inference.quantization
  • core.inference.quantization.mxfp8_quantize
  • core.inference.quantization.mxfp8_tensor
  • core.inference.quantization.utils
• core.inference.moe
  • core.inference.moe.permute
  • core.inference.moe.vllm_fused_moe
  • core.inference.moe.batch_invariant
  • core.inference.moe.metadata
  • core.inference.moe.activations
  • core.inference.moe.fused_moe
• core.inference.engines
  • core.inference.engines.static_engine
  • core.inference.engines.dynamic_engine
  • core.inference.engines.async_zmq_communicator
  • core.inference.engines.abstract_engine
  • core.inference.engines.mcore_engine
• core.inference.apis
  • core.inference.apis.llm
  • core.inference.apis._llm_base
  • core.inference.apis.serve_config
  • core.inference.apis.async_llm
• core.inference.disaggregation
  • core.inference.disaggregation.transfer_backends
    • core.inference.disaggregation.transfer_backends.nixl
    • core.inference.disaggregation.transfer_backends.nccl
    • core.inference.disaggregation.transfer_backends.base
  • core.inference.disaggregation.kv_reshard
  • core.inference.disaggregation.ssm_reshard
  • core.inference.disaggregation.utils
• core.inference.text_generation_server
  • core.inference.text_generation_server.dynamic_text_gen_server
    • core.inference.text_generation_server.dynamic_text_gen_server.endpoints
    • core.inference.text_generation_server.dynamic_text_gen_server.openai_streaming
    • core.inference.text_generation_server.dynamic_text_gen_server.text_generation_server
    • core.inference.text_generation_server.dynamic_text_gen_server.tokenization
    • core.inference.text_generation_server.dynamic_text_gen_server.incremental_detokenizer
  • core.inference.text_generation_server.run_mcore_engine
  • core.inference.text_generation_server.text_generation_server
  • core.inference.text_generation_server.tokenization
• core.inference.text_generation_controllers
  • core.inference.text_generation_controllers.mtp_utils_triton
  • core.inference.text_generation_controllers.text_generation_controller
  • core.inference.text_generation_controllers.vlm_text_generation_controller
  • core.inference.text_generation_controllers.encoder_decoder_text_generation_controller
  • core.inference.text_generation_controllers.mtp_utils_pytorch
• core.inference.model_inference_wrappers
  • core.inference.model_inference_wrappers.t5
    • core.inference.model_inference_wrappers.t5.t5_inference_wrapper
  • core.inference.model_inference_wrappers.gpt
    • core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper
  • core.inference.model_inference_wrappers.abstract_model_inference_wrapper

Submodules

• core.inference.scheduler
• core.inference.unified_memory
• core.inference.batch_dimensions_utils
• core.inference.shards_spec
• core.inference.inference_request
• core.inference.config
• core.inference.async_stream
• core.inference.communication_utils
• core.inference.sampling_params
• core.inference.symmetric_memory
• core.inference.headers
• core.inference.inference_client
• core.inference.shards
• core.inference.common_inference_params
• core.inference.utils