{
  "version": "1",
  "updated": "2026-09-27",
  "data_version": "2026-09-27",
  "unit": "Wh per million output tokens",
  "itl_deadline_ms": 500,
  "source": {
    "name": "ML.ENERGY Leaderboard",
    "publisher": "The ML.ENERGY Initiative (started at SymbioticLab, University of Michigan)",
    "url": "https://ml.energy/leaderboard/",
    "data_url": "https://raw.githubusercontent.com/ml-energy/leaderboard/master/public/data/tasks/lm-arena-chat.json",
    "version": "3.0 (released 2025-12-01); data compiled 2026-02-16",
    "released": "2025-12-01",
    "compiled": "2026-02-16",
    "compiled_evidence": "the leaderboard's public/data/index.json last_updated, and the last commit to the task files, both 2026-02-16",
    "retrieved": "2026-09-26",
    "licence": "Apache-2.0 per the ml-energy/benchmark-v3 dataset card",
    "licence_evidence": [
      "Hugging Face dataset ml-energy/benchmark-v3: tagged license:apache-2.0, ships a LICENSE file, and its dataset card's License section reads Apache-2.0",
      "The ML.ENERGY Leaderboard web app's footer says Apache 2.0",
      "The leaderboard repository (github.com/ml-energy/leaderboard) carries no licence file; its JSON files are reproduced here as published measurements until the ML.ENERGY team confirms that the dataset licence covers them"
    ],
    "workload": "Text Conversation (lm-arena-chat)",
    "method": "Energy per output token is steady-state GPU energy divided by the output tokens generated in the steady-state window (the benchmark's steady_state_energy_per_token), measured with Zeus under vLLM 0.11.1 on H100 80GB HBM3 and B200 servers. Energy per request is that figure multiplied by the average output length, not a separate measurement. Prompt processing is included.",
    "selection": "Each index row is the lowest-energy configuration for its model and GPU whose median inter-token latency does not exceed 500 ms, the leaderboard's own default ceiling; no measured configuration reaches it.",
    "cite": "Chung et al., The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization, NeurIPS 2025 Datasets and Benchmarks, arXiv 2505.06371",
    "cite_url": "https://arxiv.org/abs/2505.06371",
    "used_for": "index, per-model configurations, concurrency curves, measured GPU power",
    "configurations": "328 stable configurations: the leaderboard publishes the runs that passed its stability filter; every run, including the unstable ones, is in the gated Hugging Face dataset ml-energy/benchmark-v3"
  },
  "caveat": "GPU energy only, measured at steady state on a busy server (the most energy-efficient configuration under a latency limit). Host, network, cooling and idle time are not counted. Every figure is a floor; a lightly used server sits well above it.",
  "boundary": "gpu_only",
  "slug": "qwen-3-30b-a3b-instruct",
  "name": "Qwen 3 30B A3B Instruct",
  "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507",
  "params_b": 30.0,
  "active_params_b": 3.0,
  "architecture": "MoE",
  "gpus": [
    "B200",
    "H100"
  ],
  "configurations": [
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 7.9,
      "wh_per_million_output_tokens": 200.47,
      "joules_per_output_token": 0.7216811,
      "median_itl_ms": 8.2,
      "p90_itl_ms": 11.4,
      "p99_itl_ms": 17.2,
      "tokens_per_second": 910.0,
      "gpu_power_watts_total": 657,
      "avg_output_tokens": 915,
      "under_itl_deadline": true,
      "avg_running_batch": 7.9,
      "p95_itl_ms": 12.9,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/8",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 15.9,
      "wh_per_million_output_tokens": 148.67,
      "joules_per_output_token": 0.5352064,
      "median_itl_ms": 10.8,
      "p90_itl_ms": 20.2,
      "p99_itl_ms": 57.7,
      "tokens_per_second": 1232.0,
      "gpu_power_watts_total": 659,
      "avg_output_tokens": 911,
      "under_itl_deadline": true,
      "avg_running_batch": 15.9,
      "p95_itl_ms": 24.6,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/16",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.9,
      "wh_per_million_output_tokens": 106.29,
      "joules_per_output_token": 0.3826338,
      "median_itl_ms": 14.3,
      "p90_itl_ms": 23.6,
      "p99_itl_ms": 67.6,
      "tokens_per_second": 1940.6,
      "gpu_power_watts_total": 743,
      "avg_output_tokens": 915,
      "under_itl_deadline": true,
      "avg_running_batch": 31.9,
      "p95_itl_ms": 28.1,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/32",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.8,
      "wh_per_million_output_tokens": 69.25,
      "joules_per_output_token": 0.2492944,
      "median_itl_ms": 19.3,
      "p90_itl_ms": 31.9,
      "p99_itl_ms": 83.1,
      "tokens_per_second": 2906.6,
      "gpu_power_watts_total": 725,
      "avg_output_tokens": 929,
      "under_itl_deadline": true,
      "avg_running_batch": 63.8,
      "p95_itl_ms": 38.0,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/64",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.7,
      "wh_per_million_output_tokens": 52.09,
      "joules_per_output_token": 0.1875328,
      "median_itl_ms": 19.0,
      "p90_itl_ms": 28.8,
      "p99_itl_ms": 79.3,
      "tokens_per_second": 4400.5,
      "gpu_power_watts_total": 825,
      "avg_output_tokens": 920,
      "under_itl_deadline": true,
      "avg_running_batch": 95.7,
      "p95_itl_ms": 36.3,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/96",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.8,
      "wh_per_million_output_tokens": 43.64,
      "joules_per_output_token": 0.1571089,
      "median_itl_ms": 20.9,
      "p90_itl_ms": 30.8,
      "p99_itl_ms": 82.4,
      "tokens_per_second": 5276.5,
      "gpu_power_watts_total": 829,
      "avg_output_tokens": 898,
      "under_itl_deadline": true,
      "avg_running_batch": 127.8,
      "p95_itl_ms": 38.9,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/128",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.7,
      "wh_per_million_output_tokens": 32.27,
      "joules_per_output_token": 0.1161722,
      "median_itl_ms": 23.7,
      "p90_itl_ms": 32.7,
      "p99_itl_ms": 73.9,
      "tokens_per_second": 7212.0,
      "gpu_power_watts_total": 838,
      "avg_output_tokens": 901,
      "under_itl_deadline": true,
      "avg_running_batch": 191.7,
      "p95_itl_ms": 43.8,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/192",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.6,
      "wh_per_million_output_tokens": 28.5,
      "joules_per_output_token": 0.1025991,
      "median_itl_ms": 26.3,
      "p90_itl_ms": 35.5,
      "p99_itl_ms": 78.8,
      "tokens_per_second": 8872.0,
      "gpu_power_watts_total": 910,
      "avg_output_tokens": 914,
      "under_itl_deadline": true,
      "avg_running_batch": 255.6,
      "p95_itl_ms": 41.0,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/256",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 511.1,
      "wh_per_million_output_tokens": 22.09,
      "joules_per_output_token": 0.079536,
      "median_itl_ms": 38.0,
      "p90_itl_ms": 95.9,
      "p99_itl_ms": 113.8,
      "tokens_per_second": 8839.8,
      "gpu_power_watts_total": 703,
      "avg_output_tokens": 924,
      "under_itl_deadline": true,
      "avg_running_batch": 511.1,
      "p95_itl_ms": 101.6,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/512",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 768,
      "avg_requests_in_flight": 767.1,
      "wh_per_million_output_tokens": 18.94,
      "joules_per_output_token": 0.0681719,
      "median_itl_ms": 86.9,
      "p90_itl_ms": 112.7,
      "p99_itl_ms": 134.6,
      "tokens_per_second": 8027.9,
      "gpu_power_watts_total": 547,
      "avg_output_tokens": 922,
      "under_itl_deadline": true,
      "avg_running_batch": 767.1,
      "p95_itl_ms": 116.8,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/768",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 1022.6,
      "wh_per_million_output_tokens": 15.67,
      "joules_per_output_token": 0.0564083,
      "median_itl_ms": 88.7,
      "p90_itl_ms": 114.5,
      "p99_itl_ms": 141.1,
      "tokens_per_second": 10648.0,
      "gpu_power_watts_total": 601,
      "avg_output_tokens": 928,
      "under_itl_deadline": true,
      "avg_running_batch": 1022.6,
      "p95_itl_ms": 120.5,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/1024",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1536,
      "avg_requests_in_flight": 1478.2,
      "wh_per_million_output_tokens": 15.81,
      "joules_per_output_token": 0.0569325,
      "median_itl_ms": 123.3,
      "p90_itl_ms": 161.5,
      "p99_itl_ms": 196.0,
      "tokens_per_second": 10649.1,
      "gpu_power_watts_total": 606,
      "avg_output_tokens": 919,
      "under_itl_deadline": true,
      "avg_running_batch": 1478.2,
      "p95_itl_ms": 173.2,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/1536",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 181.43,
      "joules_per_output_token": 0.653134,
      "median_itl_ms": 12.9,
      "p90_itl_ms": 13.4,
      "p99_itl_ms": 14.2,
      "tokens_per_second": 611.3,
      "gpu_power_watts_total": 399,
      "avg_output_tokens": 927,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 13.6,
      "row_id": "qwen-3-30b-a3b-instruct/H100/1/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 135.22,
      "joules_per_output_token": 0.4867984,
      "median_itl_ms": 17.6,
      "p90_itl_ms": 18.7,
      "p99_itl_ms": 34.1,
      "tokens_per_second": 902.1,
      "gpu_power_watts_total": 439,
      "avg_output_tokens": 925,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 19.1,
      "row_id": "qwen-3-30b-a3b-instruct/H100/1/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.9,
      "wh_per_million_output_tokens": 93.25,
      "joules_per_output_token": 0.3356859,
      "median_itl_ms": 22.6,
      "p90_itl_ms": 24.1,
      "p99_itl_ms": 36.4,
      "tokens_per_second": 1393.8,
      "gpu_power_watts_total": 468,
      "avg_output_tokens": 922,
      "under_itl_deadline": true,
      "avg_running_batch": 31.9,
      "p95_itl_ms": 25.5,
      "row_id": "qwen-3-30b-a3b-instruct/H100/1/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 61.61,
      "joules_per_output_token": 0.2218075,
      "median_itl_ms": 28.9,
      "p90_itl_ms": 30.7,
      "p99_itl_ms": 44.1,
      "tokens_per_second": 2163.9,
      "gpu_power_watts_total": 480,
      "avg_output_tokens": 919,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 34.1,
      "row_id": "qwen-3-30b-a3b-instruct/H100/1/64",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 88.5,
      "wh_per_million_output_tokens": 51.63,
      "joules_per_output_token": 0.185872,
      "median_itl_ms": 32.1,
      "p90_itl_ms": 35.2,
      "p99_itl_ms": 64.8,
      "tokens_per_second": 2658.8,
      "gpu_power_watts_total": 494,
      "avg_output_tokens": 931,
      "under_itl_deadline": true,
      "avg_running_batch": 88.5,
      "p95_itl_ms": 39.9,
      "row_id": "qwen-3-30b-a3b-instruct/H100/1/96",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 228.6,
      "joules_per_output_token": 0.8229521,
      "median_itl_ms": 9.4,
      "p90_itl_ms": 9.8,
      "p99_itl_ms": 10.4,
      "tokens_per_second": 830.8,
      "gpu_power_watts_total": 684,
      "avg_output_tokens": 901,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 9.9,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 168.36,
      "joules_per_output_token": 0.6060882,
      "median_itl_ms": 12.6,
      "p90_itl_ms": 12.9,
      "p99_itl_ms": 29.6,
      "tokens_per_second": 1253.0,
      "gpu_power_watts_total": 759,
      "avg_output_tokens": 926,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 13.1,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 32.0,
      "wh_per_million_output_tokens": 114.23,
      "joules_per_output_token": 0.4112151,
      "median_itl_ms": 15.4,
      "p90_itl_ms": 16.1,
      "p99_itl_ms": 30.8,
      "tokens_per_second": 2026.0,
      "gpu_power_watts_total": 833,
      "avg_output_tokens": 927,
      "under_itl_deadline": true,
      "avg_running_batch": 32.0,
      "p95_itl_ms": 16.3,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 75.53,
      "joules_per_output_token": 0.2718903,
      "median_itl_ms": 18.7,
      "p90_itl_ms": 19.6,
      "p99_itl_ms": 34.2,
      "tokens_per_second": 3290.2,
      "gpu_power_watts_total": 895,
      "avg_output_tokens": 918,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 23.0,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/64",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.9,
      "wh_per_million_output_tokens": 64.98,
      "joules_per_output_token": 0.2339173,
      "median_itl_ms": 24.6,
      "p90_itl_ms": 25.7,
      "p99_itl_ms": 42.3,
      "tokens_per_second": 3814.1,
      "gpu_power_watts_total": 892,
      "avg_output_tokens": 921,
      "under_itl_deadline": true,
      "avg_running_batch": 95.9,
      "p95_itl_ms": 26.3,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/96",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.9,
      "wh_per_million_output_tokens": 54.97,
      "joules_per_output_token": 0.1978992,
      "median_itl_ms": 27.4,
      "p90_itl_ms": 28.8,
      "p99_itl_ms": 46.5,
      "tokens_per_second": 4546.2,
      "gpu_power_watts_total": 900,
      "avg_output_tokens": 919,
      "under_itl_deadline": true,
      "avg_running_batch": 127.9,
      "p95_itl_ms": 29.8,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/128",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.8,
      "wh_per_million_output_tokens": 43.25,
      "joules_per_output_token": 0.15569,
      "median_itl_ms": 31.3,
      "p90_itl_ms": 33.0,
      "p99_itl_ms": 52.5,
      "tokens_per_second": 5975.1,
      "gpu_power_watts_total": 930,
      "avg_output_tokens": 915,
      "under_itl_deadline": true,
      "avg_running_batch": 191.8,
      "p95_itl_ms": 34.6,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/192",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.7,
      "wh_per_million_output_tokens": 40.84,
      "joules_per_output_token": 0.1470218,
      "median_itl_ms": 33.7,
      "p90_itl_ms": 35.6,
      "p99_itl_ms": 54.1,
      "tokens_per_second": 7437.0,
      "gpu_power_watts_total": 1093,
      "avg_output_tokens": 918,
      "under_itl_deadline": true,
      "avg_running_batch": 255.7,
      "p95_itl_ms": 37.3,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/256/r2",
      "repeat": 2
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.8,
      "wh_per_million_output_tokens": 37.08,
      "joules_per_output_token": 0.1334726,
      "median_itl_ms": 34.6,
      "p90_itl_ms": 37.1,
      "p99_itl_ms": 56.7,
      "tokens_per_second": 7154.1,
      "gpu_power_watts_total": 955,
      "avg_output_tokens": 920,
      "under_itl_deadline": true,
      "avg_running_batch": 255.8,
      "p95_itl_ms": 39.9,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/256",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 384,
      "avg_requests_in_flight": 383.5,
      "wh_per_million_output_tokens": 31.11,
      "joules_per_output_token": 0.1119958,
      "median_itl_ms": 42.9,
      "p90_itl_ms": 48.1,
      "p99_itl_ms": 69.3,
      "tokens_per_second": 8387.6,
      "gpu_power_watts_total": 939,
      "avg_output_tokens": 940,
      "under_itl_deadline": true,
      "avg_running_batch": 383.5,
      "p95_itl_ms": 53.6,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/384",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 511.1,
      "wh_per_million_output_tokens": 27.44,
      "joules_per_output_token": 0.0987737,
      "median_itl_ms": 47.4,
      "p90_itl_ms": 55.8,
      "p99_itl_ms": 82.1,
      "tokens_per_second": 9650.0,
      "gpu_power_watts_total": 953,
      "avg_output_tokens": 927,
      "under_itl_deadline": true,
      "avg_running_batch": 511.1,
      "p95_itl_ms": 61.8,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/512",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 511.3,
      "wh_per_million_output_tokens": 32.64,
      "joules_per_output_token": 0.1175081,
      "median_itl_ms": 50.2,
      "p90_itl_ms": 69.3,
      "p99_itl_ms": 83.9,
      "tokens_per_second": 9366.1,
      "gpu_power_watts_total": 1101,
      "avg_output_tokens": 926,
      "under_itl_deadline": true,
      "avg_running_batch": 511.3,
      "p95_itl_ms": 75.9,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/512/r2",
      "repeat": 2
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 768,
      "avg_requests_in_flight": 767.2,
      "wh_per_million_output_tokens": 28.16,
      "joules_per_output_token": 0.1013916,
      "median_itl_ms": 65.0,
      "p90_itl_ms": 87.2,
      "p99_itl_ms": 112.7,
      "tokens_per_second": 10670.4,
      "gpu_power_watts_total": 1082,
      "avg_output_tokens": 924,
      "under_itl_deadline": true,
      "avg_running_batch": 767.2,
      "p95_itl_ms": 89.6,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/768",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 1018.8,
      "wh_per_million_output_tokens": 24.69,
      "joules_per_output_token": 0.0888773,
      "median_itl_ms": 74.8,
      "p90_itl_ms": 94.9,
      "p99_itl_ms": 134.3,
      "tokens_per_second": 12639.6,
      "gpu_power_watts_total": 1123,
      "avg_output_tokens": 920,
      "under_itl_deadline": true,
      "avg_running_batch": 1018.8,
      "p95_itl_ms": 98.3,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/1024",
      "repeat": 1
    }
  ],
  "best": {
    "B200": {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 1022.6,
      "wh_per_million_output_tokens": 15.67,
      "joules_per_output_token": 0.0564083,
      "median_itl_ms": 88.7,
      "p90_itl_ms": 114.5,
      "p99_itl_ms": 141.1,
      "tokens_per_second": 10648.0,
      "gpu_power_watts_total": 601,
      "avg_output_tokens": 928,
      "under_itl_deadline": true,
      "avg_running_batch": 1022.6,
      "p95_itl_ms": 120.5,
      "row_id": "qwen-3-30b-a3b-instruct/B200/1/1024",
      "repeat": 1
    },
    "H100": {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 1018.8,
      "wh_per_million_output_tokens": 24.69,
      "joules_per_output_token": 0.0888773,
      "median_itl_ms": 74.8,
      "p90_itl_ms": 94.9,
      "p99_itl_ms": 134.3,
      "tokens_per_second": 12639.6,
      "gpu_power_watts_total": 1123,
      "avg_output_tokens": 920,
      "under_itl_deadline": true,
      "avg_running_batch": 1018.8,
      "p95_itl_ms": 98.3,
      "row_id": "qwen-3-30b-a3b-instruct/H100/2/1024",
      "repeat": 1
    }
  }
}