dynamo.mocker

1
from dynamo.mocker import AicEngineConfig

1
AicEngineConfig(model_name: str, backend: str, system_name: str = 'h200_sxm', backend_version: Optional[str] = None, tp_size: int = 1, pp_size: int = 1, moe_tp_size: Optional[int] = None, moe_ep_size: Optional[int] = None, attention_dp_size: Optional[int] = None, model_arch: Optional[str] = None, weight_dtype: Optional[str] = None, moe_dtype: Optional[str] = None, activation_dtype: Optional[str] = None, kv_cache_dtype: Optional[str] = None, kv_block_size: Optional[int] = None, extra: Optional[dict[str, str]] = None) -> None

1
__init__(model_name: str, backend: str, system_name: str = 'h200_sxm', backend_version: Optional[str] = None, tp_size: int = 1, pp_size: int = 1, moe_tp_size: Optional[int] = None, moe_ep_size: Optional[int] = None, attention_dp_size: Optional[int] = None, model_arch: Optional[str] = None, weight_dtype: Optional[str] = None, moe_dtype: Optional[str] = None, activation_dtype: Optional[str] = None, kv_cache_dtype: Optional[str] = None, kv_block_size: Optional[int] = None, extra: Optional[dict[str, str]] = None) -> None

1
from dynamo.mocker import EngineCapacity

1
from dynamo.mocker import EngineCapacityRequest

1
EngineCapacityRequest(isl: int, osl: int, ttft_sla_ms: Optional[float] = None, itl_sla_ms: Optional[float] = None, e2e_latency_sla_ms: Optional[float] = None, kv_hit_rate: Optional[float] = None, optimization_target: OptimizationTarget = OptimizationTarget.Throughput) -> None

1
__init__(isl: int, osl: int, ttft_sla_ms: Optional[float] = None, itl_sla_ms: Optional[float] = None, e2e_latency_sla_ms: Optional[float] = None, kv_hit_rate: Optional[float] = None, optimization_target: OptimizationTarget = OptimizationTarget.Throughput) -> None

1
from dynamo.mocker import EnginePerfLimits

1
EnginePerfLimits(max_num_batched_tokens: int = 8192, max_num_seqs: int = 512, max_kv_tokens: int = 2000000) -> None

1
__init__(max_num_batched_tokens: int = 8192, max_num_seqs: int = 512, max_kv_tokens: int = 2000000) -> None

1
from dynamo.mocker import MockEngineArgs

1
MockEngineArgs(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_size: int = 0, max_num_seqs: Optional[int] = 256, max_num_batched_tokens: Optional[int] = 8192, enable_prefix_caching: bool = True, enable_chunked_prefill: bool = True, speedup_ratio: float = 1.0, decode_speedup_ratio: float = 1.0, dp_size: int = 1, startup_time: Optional[float] = None, worker_type: str = 'aggregated', planner_profile_data: Optional[str | os.PathLike[str]] = None, aic_backend: Optional[str] = None, aic_system: Optional[str] = None, aic_backend_version: Optional[str] = None, aic_tp_size: Optional[int] = None, aic_model_path: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_mtp_seed: int = 42, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None, gpu_memory_utilization: Optional[float] = None, mem_fraction_static: Optional[float] = None, free_gpu_memory_fraction: Optional[float] = None, enable_local_indexer: bool = False, bootstrap_port: Optional[int] = None, handoff_session_timeout_ms: int = 300000, kv_bytes_per_token: Optional[int] = None, kv_transfer_bandwidth: Optional[float] = None, kv_transfer_timing_mode: str = 'full_prompt', reasoning: Optional[ReasoningConfig] = None, response_replay_trace_path: Optional[str | os.PathLike[str]] = None, zmq_kv_events_port: Optional[int] = None, zmq_replay_port: Optional[int] = None, preemption_mode: str = 'lifo', router_queue_policy: Optional[str] = None, sglang: Optional[SglangArgs] = None, trtllm: Optional[TrtllmArgs] = None, num_g2_blocks: Optional[int] = None, num_g3_blocks: Optional[int] = None, offload_batch_size: Optional[int] = None, bandwidth_g1_to_g2_gbps: Optional[float] = None, bandwidth_g2_to_g1_gbps: Optional[float] = None, bandwidth_g2_to_g3_gbps: Optional[float] = None, bandwidth_g3_to_g2_gbps: Optional[float] = None, enable_g4_storage: bool = False, bandwidth_g2_to_g4_gbps: Optional[float] = None, bandwidth_g4_to_g2_gbps: Optional[float] = None, max_model_len: Optional[int] = None, g1_backend: Optional[str] = None) -> None

1
__init__(engine_type: str = 'vllm', num_gpu_blocks: Optional[int] = None, block_size: int = 0, max_num_seqs: Optional[int] = 256, max_num_batched_tokens: Optional[int] = 8192, enable_prefix_caching: bool = True, enable_chunked_prefill: bool = True, speedup_ratio: float = 1.0, decode_speedup_ratio: float = 1.0, dp_size: int = 1, startup_time: Optional[float] = None, worker_type: str = 'aggregated', planner_profile_data: Optional[str | os.PathLike[str]] = None, aic_backend: Optional[str] = None, aic_system: Optional[str] = None, aic_backend_version: Optional[str] = None, aic_tp_size: Optional[int] = None, aic_model_path: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_mtp_seed: int = 42, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None, gpu_memory_utilization: Optional[float] = None, mem_fraction_static: Optional[float] = None, free_gpu_memory_fraction: Optional[float] = None, enable_local_indexer: bool = False, bootstrap_port: Optional[int] = None, handoff_session_timeout_ms: int = 300000, kv_bytes_per_token: Optional[int] = None, kv_transfer_bandwidth: Optional[float] = None, kv_transfer_timing_mode: str = 'full_prompt', reasoning: Optional[ReasoningConfig] = None, response_replay_trace_path: Optional[str | os.PathLike[str]] = None, zmq_kv_events_port: Optional[int] = None, zmq_replay_port: Optional[int] = None, preemption_mode: str = 'lifo', router_queue_policy: Optional[str] = None, sglang: Optional[SglangArgs] = None, trtllm: Optional[TrtllmArgs] = None, num_g2_blocks: Optional[int] = None, num_g3_blocks: Optional[int] = None, offload_batch_size: Optional[int] = None, bandwidth_g1_to_g2_gbps: Optional[float] = None, bandwidth_g2_to_g1_gbps: Optional[float] = None, bandwidth_g2_to_g3_gbps: Optional[float] = None, bandwidth_g3_to_g2_gbps: Optional[float] = None, enable_g4_storage: bool = False, bandwidth_g2_to_g4_gbps: Optional[float] = None, bandwidth_g4_to_g2_gbps: Optional[float] = None, max_model_len: Optional[int] = None, g1_backend: Optional[str] = None) -> None

1
from_json(config_json: str) -> MockEngineArgs

1
copy() -> MockEngineArgs

1
is_prefill() -> bool

1
is_decode() -> bool

1
with_overrides(bootstrap_port: Optional[int] = None, zmq_kv_events_port: Optional[int] = None, zmq_replay_port: Optional[int] = None, kv_bytes_per_token: Optional[int] = None, num_gpu_blocks: Optional[int] = None, aic_backend: Optional[str] = None, aic_system: Optional[str] = None, aic_backend_version: Optional[str] = None, aic_tp_size: Optional[int] = None, aic_model_path: Optional[str] = None, aic_moe_tp_size: Optional[int] = None, aic_moe_ep_size: Optional[int] = None, aic_attention_dp_size: Optional[int] = None, aic_nextn: Optional[int] = None, aic_nextn_accept_rates: Optional[str] = None, aic_mtp_seed: Optional[int] = None, aic_gemm_dtype: Optional[str] = None, aic_moe_dtype: Optional[str] = None, aic_fmha_dtype: Optional[str] = None, aic_kv_cache_dtype: Optional[str] = None, aic_comm_dtype: Optional[str] = None, gpu_memory_utilization: Optional[float] = None, mem_fraction_static: Optional[float] = None, free_gpu_memory_fraction: Optional[float] = None, enable_prefix_caching: Optional[bool] = None, worker_type: Optional[str] = None) -> MockEngineArgs

1
from dynamo.mocker import OptimizationTarget

1
from dynamo.mocker.args import ProfileDataResult

1
ProfileDataResult(npz_path: Path | None, tmpdir: tempfile.TemporaryDirectory | None)

1
__init__(npz_path: Path | None, tmpdir: tempfile.TemporaryDirectory | None)

1
from dynamo.mocker import ReasoningConfig

1
ReasoningConfig(start_thinking_token_id: int, end_thinking_token_id: int, thinking_ratio: float) -> None

1
__init__(start_thinking_token_id: int, end_thinking_token_id: int, thinking_ratio: float) -> None

1
from dynamo.mocker import RustEnginePerfModel

1
best_available(*, engine_args: Optional[MockEngineArgs] = None, aic_config: Optional[AicEngineConfig] = None, worker_type: Optional[str] = None, limits: Optional[EnginePerfLimits] = None, options: Optional[RustEnginePerfOptions] = None, bootstrap_fpms: Optional[Any] = None) -> RustEnginePerfModel

1
from_regression(*, worker_type: str, limits: EnginePerfLimits, options: Optional[RustEnginePerfOptions] = None, bootstrap_fpms: Optional[Any] = None) -> RustEnginePerfModel

1
from_native(*, aic_config: AicEngineConfig, worker_type: str, limits: EnginePerfLimits, options: Optional[RustEnginePerfOptions] = None, bootstrap_fpms: Optional[Any] = None) -> RustEnginePerfModel

1
estimate_forward_pass_time(metrics_by_rank: Any) -> Optional[float]

1
tune_with_fpms(iterations: Any) -> None

1
diagnostics() -> str

1
get_min_correction_factor() -> Optional[float]

1
get_max_correction_factor() -> Optional[float]

1
get_avg_correction_factor() -> Optional[float]

1
get_queued_prefill_time(metrics_by_rank: Any) -> Optional[float]

1
get_scheduled_decode_itl(metrics_by_rank: Any) -> Optional[float]

1
find_engine_capacity_rps(request: EngineCapacityRequest) -> Optional[EngineCapacity]

1
from dynamo.mocker import RustEnginePerfOptions

1
RustEnginePerfOptions(max_observations: int = 64, min_observations: int = 5, bucket_count: int = 16, max_num_tokens: int = 8192, max_batch_size: int = 512, max_kv_tokens: int = 2000000) -> None

1
__init__(max_observations: int = 64, min_observations: int = 5, bucket_count: int = 16, max_num_tokens: int = 8192, max_batch_size: int = 512, max_kv_tokens: int = 2000000) -> None

1
from dynamo.mocker import SglangArgs

1
SglangArgs(schedule_policy: Optional[str] = None, page_size: Optional[int] = None, max_prefill_tokens: Optional[int] = None, chunked_prefill_size: Optional[int] = None, clip_max_new_tokens: Optional[int] = None, schedule_conservativeness: Optional[float] = None) -> None

1
__init__(schedule_policy: Optional[str] = None, page_size: Optional[int] = None, max_prefill_tokens: Optional[int] = None, chunked_prefill_size: Optional[int] = None, clip_max_new_tokens: Optional[int] = None, schedule_conservativeness: Optional[float] = None) -> None

1
from dynamo.mocker import TrtllmArgs

1
TrtllmArgs(capacity_scheduler_policy: Optional[str] = None) -> None

1
__init__(capacity_scheduler_policy: Optional[str] = None) -> None

1
from dynamo.mocker.config import apply_worker_engine_args_overrides

1
apply_worker_engine_args_overrides(engine_args: MockEngineArgs, *, kv_bytes_per_token: int | None = None, bootstrap_port: int | None = None, zmq_kv_events_port: int | None = None, zmq_replay_port: int | None = None, aic_mtp_seed: int | None = None) -> MockEngineArgs

1
from dynamo.mocker.config import build_mocker_engine_args

1
build_mocker_engine_args(args: argparse.Namespace) -> MockEngineArgs

1
from dynamo.mocker.config import build_runtime_config

1
build_runtime_config(engine_args: MockEngineArgs) -> tuple[int, ModelRuntimeConfig]

1
from dynamo.mocker.main import compute_stagger_delay

1
compute_stagger_delay(num_workers: int, stagger_delay: float) -> float

1
from dynamo.mocker.main import graceful_shutdown

1
graceful_shutdown(runtimes: list)

1
from dynamo.mocker.main import launch_workers

1
launch_workers(args: argparse.Namespace, base_engine_args)

1
from dynamo.mocker.config import load_mocker_engine_args

1
load_mocker_engine_args(args: argparse.Namespace) -> MockEngineArgs

1
from dynamo.mocker.main import main

1
main()

1
from dynamo.mocker.args import non_negative_float

1
non_negative_float(value: str) -> float

1
from dynamo.mocker.args import non_negative_int

1
non_negative_int(value: str) -> int

1
from dynamo.mocker.args import parse_args

1
parse_args(argv: list[str] | None = None) -> argparse.Namespace

1
from dynamo.mocker.args import parse_bootstrap_ports

1
parse_bootstrap_ports(ports_str: str | None) -> list[int]

1
from dynamo.mocker.args import positive_int

1
positive_int(value: str) -> int

1
from dynamo.mocker.main import prefetch_model

1
prefetch_model(model_path: str) -> None

1
from dynamo.mocker.args import resolve_planner_profile_data

1
resolve_planner_profile_data(planner_profile_data: Path | None) -> ProfileDataResult

1
from dynamo.mocker import run_mocker_trace_replay

1
run_mocker_trace_replay(trace_files, extra_engine_args = None, router_config = None, num_workers = 1, replay_concurrency = None, router_mode = 'round_robin', arrival_speedup_ratio = 1.0, trace_block_size = None, trace_format = 'mooncake', trace_shared_prefix_ratio = 0.0, trace_num_prefix_groups = 0, model_name = None, sla_ttft_ms = None, sla_itl_ms = None, sla_e2e_ms = None)

1
from dynamo.mocker.args import validate_worker_type_args

1
validate_worker_type_args(args: argparse.Namespace) -> None

1
from dynamo.mocker.main import worker

1
worker()