core.inference

Subpackages

• core.inference.text_generation_controllers
  • core.inference.text_generation_controllers.vlm_text_generation_controller
  • core.inference.text_generation_controllers.encoder_decoder_text_generation_controller
  • core.inference.text_generation_controllers.text_generation_controller
• core.inference.model_inference_wrappers
  • core.inference.model_inference_wrappers.gpt
    • core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper
  • core.inference.model_inference_wrappers.t5
    • core.inference.model_inference_wrappers.t5.t5_inference_wrapper
  • core.inference.model_inference_wrappers.abstract_model_inference_wrapper
• core.inference.contexts
  • core.inference.contexts.base_context
  • core.inference.contexts.routing_metadata
  • core.inference.contexts.dynamic_block_allocator
  • core.inference.contexts.static_context
  • core.inference.contexts.dynamic_context
  • core.inference.contexts.fused_kv_append_kernel
• core.inference.text_generation_server
  • core.inference.text_generation_server.dynamic_text_gen_server
    • core.inference.text_generation_server.dynamic_text_gen_server.endpoints
    • core.inference.text_generation_server.dynamic_text_gen_server.tokenization
    • core.inference.text_generation_server.dynamic_text_gen_server.flask_server
  • core.inference.text_generation_server.text_generation_server
  • core.inference.text_generation_server.tokenization
  • core.inference.text_generation_server.run_mcore_engine
• core.inference.engines
  • core.inference.engines.abstract_engine
  • core.inference.engines.mcore_engine
  • core.inference.engines.async_zmq_communicator
  • core.inference.engines.dynamic_engine
  • core.inference.engines.static_engine

Submodules

• core.inference.batch_dimensions_utils
• core.inference.inference_client
• core.inference.communication_utils
• core.inference.data_parallel_inference_coordinator
• core.inference.unified_memory
• core.inference.common_inference_params
• core.inference.inference_request
• core.inference.config
• core.inference.headers
• core.inference.scheduler
• core.inference.async_stream
• core.inference.sampling_params
• core.inference.utils