{
  "version": "1",
  "updated": "2026-09-27",
  "data_version": "2026-09-27",
  "unit": "Wh per million output tokens",
  "itl_deadline_ms": 500,
  "source": {
    "name": "ML.ENERGY Leaderboard",
    "publisher": "The ML.ENERGY Initiative (started at SymbioticLab, University of Michigan)",
    "url": "https://ml.energy/leaderboard/",
    "data_url": "https://raw.githubusercontent.com/ml-energy/leaderboard/master/public/data/tasks/lm-arena-chat.json",
    "version": "3.0 (released 2025-12-01); data compiled 2026-02-16",
    "released": "2025-12-01",
    "compiled": "2026-02-16",
    "compiled_evidence": "the leaderboard's public/data/index.json last_updated, and the last commit to the task files, both 2026-02-16",
    "retrieved": "2026-09-26",
    "licence": "Apache-2.0 per the ml-energy/benchmark-v3 dataset card",
    "licence_evidence": [
      "Hugging Face dataset ml-energy/benchmark-v3: tagged license:apache-2.0, ships a LICENSE file, and its dataset card's License section reads Apache-2.0",
      "The ML.ENERGY Leaderboard web app's footer says Apache 2.0",
      "The leaderboard repository (github.com/ml-energy/leaderboard) carries no licence file; its JSON files are reproduced here as published measurements until the ML.ENERGY team confirms that the dataset licence covers them"
    ],
    "workload": "Text Conversation (lm-arena-chat)",
    "method": "Energy per output token is steady-state GPU energy divided by the output tokens generated in the steady-state window (the benchmark's steady_state_energy_per_token), measured with Zeus under vLLM 0.11.1 on H100 80GB HBM3 and B200 servers. Energy per request is that figure multiplied by the average output length, not a separate measurement. Prompt processing is included.",
    "selection": "Each index row is the lowest-energy configuration for its model and GPU whose median inter-token latency does not exceed 500 ms, the leaderboard's own default ceiling; no measured configuration reaches it.",
    "cite": "Chung et al., The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization, NeurIPS 2025 Datasets and Benchmarks, arXiv 2505.06371",
    "cite_url": "https://arxiv.org/abs/2505.06371",
    "used_for": "index, per-model configurations, concurrency curves, measured GPU power",
    "configurations": "328 stable configurations: the leaderboard publishes the runs that passed its stability filter; every run, including the unstable ones, is in the gated Hugging Face dataset ml-energy/benchmark-v3"
  },
  "caveat": "GPU energy only, measured at steady state on a busy server (the most energy-efficient configuration under a latency limit). Host, network, cooling and idle time are not counted. Every figure is a floor; a lightly used server sits well above it.",
  "boundary": "gpu_only",
  "slug": "qwen-3-32b",
  "name": "Qwen 3 32B",
  "model_id": "Qwen/Qwen3-32B",
  "params_b": 32.0,
  "active_params_b": 32.0,
  "architecture": "Dense Transformer",
  "gpus": [
    "B200",
    "H100"
  ],
  "configurations": [
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 429.18,
      "joules_per_output_token": 1.5450541,
      "median_itl_ms": 14.1,
      "p90_itl_ms": 15.7,
      "p99_itl_ms": 27.1,
      "tokens_per_second": 556.4,
      "gpu_power_watts_total": 860,
      "avg_output_tokens": 646,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 17.3,
      "row_id": "qwen-3-32b/B200/1/8",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 226.65,
      "joules_per_output_token": 0.8159244,
      "median_itl_ms": 15.1,
      "p90_itl_ms": 18.0,
      "p99_itl_ms": 31.6,
      "tokens_per_second": 1025.1,
      "gpu_power_watts_total": 836,
      "avg_output_tokens": 629,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 20.8,
      "row_id": "qwen-3-32b/B200/1/16",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.9,
      "wh_per_million_output_tokens": 131.53,
      "joules_per_output_token": 0.4734976,
      "median_itl_ms": 16.8,
      "p90_itl_ms": 24.1,
      "p99_itl_ms": 40.1,
      "tokens_per_second": 1792.6,
      "gpu_power_watts_total": 849,
      "avg_output_tokens": 634,
      "under_itl_deadline": true,
      "avg_running_batch": 31.9,
      "p95_itl_ms": 27.7,
      "row_id": "qwen-3-32b/B200/1/32",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 86.84,
      "joules_per_output_token": 0.3126325,
      "median_itl_ms": 19.6,
      "p90_itl_ms": 27.6,
      "p99_itl_ms": 48.9,
      "tokens_per_second": 3024.1,
      "gpu_power_watts_total": 945,
      "avg_output_tokens": 631,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 31.9,
      "row_id": "qwen-3-32b/B200/1/64",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.8,
      "wh_per_million_output_tokens": 65.72,
      "joules_per_output_token": 0.2365772,
      "median_itl_ms": 21.3,
      "p90_itl_ms": 28.9,
      "p99_itl_ms": 50.9,
      "tokens_per_second": 4102.1,
      "gpu_power_watts_total": 970,
      "avg_output_tokens": 629,
      "under_itl_deadline": true,
      "avg_running_batch": 95.8,
      "p95_itl_ms": 34.0,
      "row_id": "qwen-3-32b/B200/1/96",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.8,
      "wh_per_million_output_tokens": 57.96,
      "joules_per_output_token": 0.2086465,
      "median_itl_ms": 24.7,
      "p90_itl_ms": 32.9,
      "p99_itl_ms": 65.6,
      "tokens_per_second": 4675.8,
      "gpu_power_watts_total": 976,
      "avg_output_tokens": 634,
      "under_itl_deadline": true,
      "avg_running_batch": 127.8,
      "p95_itl_ms": 38.2,
      "row_id": "qwen-3-32b/B200/1/128",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.6,
      "wh_per_million_output_tokens": 49.12,
      "joules_per_output_token": 0.1768146,
      "median_itl_ms": 31.6,
      "p90_itl_ms": 41.8,
      "p99_itl_ms": 83.9,
      "tokens_per_second": 5482.7,
      "gpu_power_watts_total": 969,
      "avg_output_tokens": 635,
      "under_itl_deadline": true,
      "avg_running_batch": 191.6,
      "p95_itl_ms": 51.1,
      "row_id": "qwen-3-32b/B200/1/192",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.6,
      "wh_per_million_output_tokens": 41.87,
      "joules_per_output_token": 0.1507231,
      "median_itl_ms": 36.0,
      "p90_itl_ms": 48.7,
      "p99_itl_ms": 97.6,
      "tokens_per_second": 6444.0,
      "gpu_power_watts_total": 971,
      "avg_output_tokens": 627,
      "under_itl_deadline": true,
      "avg_running_batch": 255.6,
      "p95_itl_ms": 56.4,
      "row_id": "qwen-3-32b/B200/1/256",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 384,
      "avg_requests_in_flight": 383.3,
      "wh_per_million_output_tokens": 39.05,
      "joules_per_output_token": 0.1405812,
      "median_itl_ms": 50.2,
      "p90_itl_ms": 66.6,
      "p99_itl_ms": 131.6,
      "tokens_per_second": 6911.8,
      "gpu_power_watts_total": 972,
      "avg_output_tokens": 635,
      "under_itl_deadline": true,
      "avg_running_batch": 383.3,
      "p95_itl_ms": 77.1,
      "row_id": "qwen-3-32b/B200/1/384",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 455.7,
      "wh_per_million_output_tokens": 41.89,
      "joules_per_output_token": 0.1507861,
      "median_itl_ms": 57.6,
      "p90_itl_ms": 93.0,
      "p99_itl_ms": 250.0,
      "tokens_per_second": 6425.5,
      "gpu_power_watts_total": 969,
      "avg_output_tokens": 627,
      "under_itl_deadline": true,
      "avg_running_batch": 455.7,
      "p95_itl_ms": 131.6,
      "row_id": "qwen-3-32b/B200/1/512",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 479.35,
      "joules_per_output_token": 1.7256602,
      "median_itl_ms": 25.2,
      "p90_itl_ms": 26.0,
      "p99_itl_ms": 27.4,
      "tokens_per_second": 312.0,
      "gpu_power_watts_total": 538,
      "avg_output_tokens": 638,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 26.4,
      "row_id": "qwen-3-32b/H100/1/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 267.04,
      "joules_per_output_token": 0.9613599,
      "median_itl_ms": 26.4,
      "p90_itl_ms": 27.7,
      "p99_itl_ms": 33.4,
      "tokens_per_second": 589.0,
      "gpu_power_watts_total": 566,
      "avg_output_tokens": 626,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 28.3,
      "row_id": "qwen-3-32b/H100/1/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 29.3,
      "wh_per_million_output_tokens": 173.33,
      "joules_per_output_token": 0.6239902,
      "median_itl_ms": 28.8,
      "p90_itl_ms": 29.9,
      "p99_itl_ms": 67.2,
      "tokens_per_second": 960.5,
      "gpu_power_watts_total": 599,
      "avg_output_tokens": 629,
      "under_itl_deadline": true,
      "avg_running_batch": 29.3,
      "p95_itl_ms": 30.7,
      "row_id": "qwen-3-32b/H100/1/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 451.98,
      "joules_per_output_token": 1.6271197,
      "median_itl_ms": 15.1,
      "p90_itl_ms": 15.5,
      "p99_itl_ms": 17.3,
      "tokens_per_second": 522.5,
      "gpu_power_watts_total": 850,
      "avg_output_tokens": 631,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 15.6,
      "row_id": "qwen-3-32b/H100/2/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 252.76,
      "joules_per_output_token": 0.9099359,
      "median_itl_ms": 16.0,
      "p90_itl_ms": 16.6,
      "p99_itl_ms": 21.2,
      "tokens_per_second": 970.2,
      "gpu_power_watts_total": 883,
      "avg_output_tokens": 626,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 16.8,
      "row_id": "qwen-3-32b/H100/2/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.9,
      "wh_per_million_output_tokens": 153.77,
      "joules_per_output_token": 0.5535648,
      "median_itl_ms": 17.9,
      "p90_itl_ms": 18.7,
      "p99_itl_ms": 33.9,
      "tokens_per_second": 1706.1,
      "gpu_power_watts_total": 944,
      "avg_output_tokens": 642,
      "under_itl_deadline": true,
      "avg_running_batch": 31.9,
      "p95_itl_ms": 19.1,
      "row_id": "qwen-3-32b/H100/2/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 101.96,
      "joules_per_output_token": 0.3670707,
      "median_itl_ms": 22.4,
      "p90_itl_ms": 23.6,
      "p99_itl_ms": 65.0,
      "tokens_per_second": 2653.8,
      "gpu_power_watts_total": 974,
      "avg_output_tokens": 635,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 25.7,
      "row_id": "qwen-3-32b/H100/2/64",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.8,
      "wh_per_million_output_tokens": 84.6,
      "joules_per_output_token": 0.3045544,
      "median_itl_ms": 26.1,
      "p90_itl_ms": 28.7,
      "p99_itl_ms": 78.1,
      "tokens_per_second": 3355.6,
      "gpu_power_watts_total": 1022,
      "avg_output_tokens": 635,
      "under_itl_deadline": true,
      "avg_running_batch": 95.8,
      "p95_itl_ms": 32.1,
      "row_id": "qwen-3-32b/H100/2/96",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.8,
      "wh_per_million_output_tokens": 76.37,
      "joules_per_output_token": 0.2749347,
      "median_itl_ms": 29.7,
      "p90_itl_ms": 33.4,
      "p99_itl_ms": 97.0,
      "tokens_per_second": 3905.3,
      "gpu_power_watts_total": 1074,
      "avg_output_tokens": 624,
      "under_itl_deadline": true,
      "avg_running_batch": 127.8,
      "p95_itl_ms": 42.7,
      "row_id": "qwen-3-32b/H100/2/128",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.7,
      "wh_per_million_output_tokens": 69.01,
      "joules_per_output_token": 0.2484368,
      "median_itl_ms": 37.0,
      "p90_itl_ms": 43.8,
      "p99_itl_ms": 129.0,
      "tokens_per_second": 4601.3,
      "gpu_power_watts_total": 1143,
      "avg_output_tokens": 632,
      "under_itl_deadline": true,
      "avg_running_batch": 191.7,
      "p95_itl_ms": 61.6,
      "row_id": "qwen-3-32b/H100/2/192",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.6,
      "wh_per_million_output_tokens": 63.47,
      "joules_per_output_token": 0.2284806,
      "median_itl_ms": 44.5,
      "p90_itl_ms": 55.6,
      "p99_itl_ms": 148.2,
      "tokens_per_second": 5155.5,
      "gpu_power_watts_total": 1178,
      "avg_output_tokens": 634,
      "under_itl_deadline": true,
      "avg_running_batch": 255.6,
      "p95_itl_ms": 74.4,
      "row_id": "qwen-3-32b/H100/2/256",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 384,
      "avg_requests_in_flight": 340.2,
      "wh_per_million_output_tokens": 64.64,
      "joules_per_output_token": 0.232693,
      "median_itl_ms": 54.9,
      "p90_itl_ms": 82.2,
      "p99_itl_ms": 240.2,
      "tokens_per_second": 5070.8,
      "gpu_power_watts_total": 1180,
      "avg_output_tokens": 628,
      "under_itl_deadline": true,
      "avg_running_batch": 340.2,
      "p95_itl_ms": 111.2,
      "row_id": "qwen-3-32b/H100/2/384",
      "repeat": 1
    }
  ],
  "best": {
    "B200": {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 384,
      "avg_requests_in_flight": 383.3,
      "wh_per_million_output_tokens": 39.05,
      "joules_per_output_token": 0.1405812,
      "median_itl_ms": 50.2,
      "p90_itl_ms": 66.6,
      "p99_itl_ms": 131.6,
      "tokens_per_second": 6911.8,
      "gpu_power_watts_total": 972,
      "avg_output_tokens": 635,
      "under_itl_deadline": true,
      "avg_running_batch": 383.3,
      "p95_itl_ms": 77.1,
      "row_id": "qwen-3-32b/B200/1/384",
      "repeat": 1
    },
    "H100": {
      "gpu": "H100",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.6,
      "wh_per_million_output_tokens": 63.47,
      "joules_per_output_token": 0.2284806,
      "median_itl_ms": 44.5,
      "p90_itl_ms": 55.6,
      "p99_itl_ms": 148.2,
      "tokens_per_second": 5155.5,
      "gpu_power_watts_total": 1178,
      "avg_output_tokens": 634,
      "under_itl_deadline": true,
      "avg_running_batch": 255.6,
      "p95_itl_ms": 74.4,
      "row_id": "qwen-3-32b/H100/2/256",
      "repeat": 1
    }
  }
}