Sweep Aggregate API Reference

1
import json
2
from pathlib import Path
3
4
# Load sweep aggregate
5
sweep_file = Path("artifacts/my_benchmark/sweep_aggregate/profile_export_aiperf_sweep.json")
6
with open(sweep_file) as f:
7
    sweep_data = json.load(f)
8
9
# Inspect metadata
10
metadata = sweep_data["metadata"]
11
sweep_params = metadata["sweep_parameters"]
12
print(f"Sweep parameters: {[p['name'] for p in sweep_params]}")
13
print(f"Total combinations: {metadata['num_combinations']}")
14
print(f"Total runs: {sweep_data['num_profile_runs']}")

1
# Get best configurations
2
best_configs = sweep_data["best_configurations"]
3
4
best_throughput = best_configs["best_throughput"]
5
print(f"Best throughput: {best_throughput['metric']:.2f} {best_throughput['unit']}")
6
print(f"  Parameters: {best_throughput['parameters']}")
7
8
best_latency = best_configs["best_latency_p99"]
9
print(f"Best latency: {best_latency['metric']:.2f} {best_latency['unit']}")
10
print(f"  Parameters: {best_latency['parameters']}")

1
# Get Pareto optimal points
2
pareto_optimal = sweep_data["pareto_optimal"]
3
print(f"Found {len(pareto_optimal)} Pareto optimal configurations")
4
5
# Extract metrics for Pareto points
6
per_combination_metrics = sweep_data["per_combination_metrics"]
7
8
print("\nPareto Frontier:")
9
for combo in per_combination_metrics:
10
    params = combo["parameters"]
11
    # Check if this combination is Pareto optimal
12
    if params in pareto_optimal:
13
        metrics = combo["metrics"]
14
        throughput = metrics["request_throughput_avg"]["mean"]
15
        latency = metrics["time_to_first_token_p99"]["mean"]
16
        print(f"  {params}: {throughput:.1f} req/s, {latency:.1f} ms p99")

1
import matplotlib.pyplot as plt
2
3
# Extract data for single-parameter sweep
4
combinations = sweep_data["per_combination_metrics"]
5
6
# Assuming single parameter (concurrency)
7
param_name = sweep_data["metadata"]["sweep_parameters"][0]["name"]
8
param_values = []
9
throughputs = []
10
ci_lows = []
11
ci_highs = []
12
13
for combo in combinations:
14
    param_value = combo["parameters"][param_name]
15
    tp = combo["metrics"]["request_throughput_avg"]
16
17
    param_values.append(param_value)
18
    throughputs.append(tp["mean"])
19
    ci_lows.append(tp.get("ci_low", tp["mean"]))
20
    ci_highs.append(tp.get("ci_high", tp["mean"]))
21
22
# Plot with confidence intervals
23
plt.figure(figsize=(10, 6))
24
plt.plot(param_values, throughputs, 'o-', label='Mean Throughput')
25
plt.fill_between(param_values, ci_lows, ci_highs, alpha=0.3, label='95% CI')
26
plt.xlabel(param_name.title())
27
plt.ylabel('Throughput (requests/sec)')
28
plt.title(f'Throughput vs {param_name.title()}')
29
plt.legend()
30
plt.grid(True)
31
plt.savefig('throughput_sweep.png')

1
import pandas as pd
2
3
# Convert per-combination metrics to DataFrame
4
rows = []
5
for combo in sweep_data["per_combination_metrics"]:
6
    row = combo["parameters"].copy()
7
8
    # Add metrics
9
    for metric_name, metric_data in combo["metrics"].items():
10
        if isinstance(metric_data, dict):
11
            row[f"{metric_name}_mean"] = metric_data.get("mean")
12
            row[f"{metric_name}_std"] = metric_data.get("std")
13
            row[f"{metric_name}_cv"] = metric_data.get("cv")
14
        else:
15
            row[metric_name] = metric_data
16
    rows.append(row)
17
18
df = pd.DataFrame(rows)
19
20
# Sort by parameter values
21
param_names = [p["name"] for p in sweep_data["metadata"]["sweep_parameters"]]
22
df = df.sort_values(param_names)
23
24
# Analyze
25
print(df[[*param_names, "request_throughput_avg_mean", "time_to_first_token_p99_mean"]])
26
27
# Export
28
df.to_csv("sweep_analysis.csv", index=False)

1
# For sweeps with multiple parameters
2
sweep_params = sweep_data["metadata"]["sweep_parameters"]
3
param_names = [p["name"] for p in sweep_params]
4
5
print(f"Multi-parameter sweep: {', '.join(param_names)}")
6
7
# Find best combination for each parameter individually
8
for param_name in param_names:
9
    # Group by this parameter
10
    param_groups = {}
11
    for combo in sweep_data["per_combination_metrics"]:
12
        param_value = combo["parameters"][param_name]
13
        if param_value not in param_groups:
14
            param_groups[param_value] = []
15
        param_groups[param_value].append(combo)
16
17
    # Find best throughput for each value of this parameter
18
    print(f"\nBest throughput for each {param_name}:")
19
    for value, combos in sorted(param_groups.items()):
20
        best_combo = max(combos,
21
                        key=lambda c: c["metrics"]["request_throughput_avg"]["mean"])
22
        throughput = best_combo["metrics"]["request_throughput_avg"]["mean"]
23
        print(f"  {param_name}={value}: {throughput:.1f} req/s")
24
        print(f"    Full config: {best_combo['parameters']}")

1
# For single-parameter sweeps, calculate efficiency
2
combinations = sweep_data["per_combination_metrics"]
3
param_name = sweep_data["metadata"]["sweep_parameters"][0]["name"]
4
5
# Sort by parameter value
6
combinations_sorted = sorted(combinations,
7
                            key=lambda c: c["parameters"][param_name])
8
9
efficiencies = []
10
for combo in combinations_sorted:
11
    param_value = combo["parameters"][param_name]
12
    throughput = combo["metrics"]["request_throughput_avg"]["mean"]
13
    efficiency = throughput / param_value
14
    efficiencies.append((param_value, efficiency))
15
16
# Find point of diminishing returns (where efficiency drops significantly)
17
threshold = 0.8  # 20% drop
18
for i in range(1, len(efficiencies)):
19
    if efficiencies[i][1] < threshold * efficiencies[i-1][1]:
20
        print(f"Diminishing returns detected at {param_name}={efficiencies[i][0]}")
21
        print(f"  Efficiency dropped from {efficiencies[i-1][1]:.2f} to {efficiencies[i][1]:.2f}")
22
        break

1
# Score configurations based on weighted objectives
2
weights = {
3
    "throughput": 0.6,  # 60% weight on throughput
4
    "latency": 0.4,     # 40% weight on latency
5
}
6
7
# Extract all throughputs and latencies
8
combinations = sweep_data["per_combination_metrics"]
9
throughputs = [c["metrics"]["request_throughput_avg"]["mean"] for c in combinations]
10
latencies = [c["metrics"]["time_to_first_token_p99"]["mean"] for c in combinations]
11
12
max_tp = max(throughputs)
13
min_lat = min(latencies)
14
max_lat = max(latencies)
15
16
scores = []
17
for combo in combinations:
18
    tp = combo["metrics"]["request_throughput_avg"]["mean"]
19
    lat = combo["metrics"]["time_to_first_token_p99"]["mean"]
20
21
    # Normalize: higher is better for both
22
    tp_score = tp / max_tp
23
    lat_score = 1 - (lat - min_lat) / (max_lat - min_lat) if max_lat > min_lat else 1.0
24
25
    # Weighted combination
26
    score = weights["throughput"] * tp_score + weights["latency"] * lat_score
27
    scores.append((combo["parameters"], score))
28
29
# Find best configuration
30
best_params, best_score = max(scores, key=lambda x: x[1])
31
print(f"Best configuration for given weights: {best_params}")
32
print(f"  Score: {best_score:.3f}")