core.inference.disaggregation.pending_handoff_imports

State records for KV-cache and SSM-state imports awaiting completion.

Module Contents

Classes

DeferredKvHandoff	Decode handoff waiting for local cache capacity before transfer starts.
PendingSSMImport	Exact SSM state being transferred into a reserved live request slot.
PendingKvImport	Decode request waiting for an asynchronous KV-cache import.

API

class core.inference.disaggregation.pending_handoff_imports.DeferredKvHandoff

Decode handoff waiting for local cache capacity before transfer starts.

request_id: int

None

prompt: list

None

sampling_params: megatron.core.inference.sampling_params.SamplingParams

None

kv_meta: dict

None

src_block_ids: List[int]

None

hashes: List[int]

None

num_blocks: int

None

future: asyncio.Future

None

class core.inference.disaggregation.pending_handoff_imports.PendingSSMImport

Exact SSM state being transferred into a reserved live request slot.

handles: list[Any]

None

live_slot: int

None

class core.inference.disaggregation.pending_handoff_imports.PendingKvImport

Decode request waiting for an asynchronous KV-cache import.

request_id: int

None

prompt: list

None

sampling_params: megatron.core.inference.sampling_params.SamplingParams

None

local_blocks: List[int]

None

hashes: List[int]

None

cached_prefix_block_count: int

None

handle: Any

None

future: asyncio.Future

None

ssm: core.inference.disaggregation.pending_handoff_imports.PendingSSMImport | None

None

resume_tokens: List[int]

‘field(…)’

continuation_blocks: List[int]

‘field(…)’

local_error: Exception | None

None

destinations_safe: bool

True

terminal_state_reported: bool

False