{
  "version": "1",
  "updated": "2026-09-27",
  "data_version": "2026-09-27",
  "unit": "Wh per million output tokens",
  "itl_deadline_ms": 500,
  "source": {
    "name": "ML.ENERGY Leaderboard",
    "publisher": "The ML.ENERGY Initiative (started at SymbioticLab, University of Michigan)",
    "url": "https://ml.energy/leaderboard/",
    "data_url": "https://raw.githubusercontent.com/ml-energy/leaderboard/master/public/data/tasks/lm-arena-chat.json",
    "version": "3.0 (released 2025-12-01); data compiled 2026-02-16",
    "released": "2025-12-01",
    "compiled": "2026-02-16",
    "compiled_evidence": "the leaderboard's public/data/index.json last_updated, and the last commit to the task files, both 2026-02-16",
    "retrieved": "2026-09-26",
    "licence": "Apache-2.0 per the ml-energy/benchmark-v3 dataset card",
    "licence_evidence": [
      "Hugging Face dataset ml-energy/benchmark-v3: tagged license:apache-2.0, ships a LICENSE file, and its dataset card's License section reads Apache-2.0",
      "The ML.ENERGY Leaderboard web app's footer says Apache 2.0",
      "The leaderboard repository (github.com/ml-energy/leaderboard) carries no licence file; its JSON files are reproduced here as published measurements until the ML.ENERGY team confirms that the dataset licence covers them"
    ],
    "workload": "Text Conversation (lm-arena-chat)",
    "method": "Energy per output token is steady-state GPU energy divided by the output tokens generated in the steady-state window (the benchmark's steady_state_energy_per_token), measured with Zeus under vLLM 0.11.1 on H100 80GB HBM3 and B200 servers. Energy per request is that figure multiplied by the average output length, not a separate measurement. Prompt processing is included.",
    "selection": "Each index row is the lowest-energy configuration for its model and GPU whose median inter-token latency does not exceed 500 ms, the leaderboard's own default ceiling; no measured configuration reaches it.",
    "cite": "Chung et al., The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization, NeurIPS 2025 Datasets and Benchmarks, arXiv 2505.06371",
    "cite_url": "https://arxiv.org/abs/2505.06371",
    "used_for": "index, per-model configurations, concurrency curves, measured GPU power",
    "configurations": "328 stable configurations: the leaderboard publishes the runs that passed its stability filter; every run, including the unstable ones, is in the gated Hugging Face dataset ml-energy/benchmark-v3"
  },
  "caveat": "GPU energy only, measured at steady state on a busy server (the most energy-efficient configuration under a latency limit). Host, network, cooling and idle time are not counted. Every figure is a floor; a lightly used server sits well above it.",
  "boundary": "gpu_only",
  "slug": "qwen-3-235b-a22b-instruct-fp8",
  "name": "Qwen 3 235B A22B Instruct FP8",
  "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8",
  "params_b": 235.0,
  "active_params_b": 22.0,
  "architecture": "MoE",
  "gpus": [
    "B200",
    "H100"
  ],
  "configurations": [
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "FP8",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 2339.19,
      "joules_per_output_token": 8.421096,
      "median_itl_ms": 36.7,
      "p90_itl_ms": 37.8,
      "p99_itl_ms": 38.6,
      "tokens_per_second": 214.4,
      "gpu_power_watts_total": 1806,
      "avg_output_tokens": 875,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 38.0,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/8",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "FP8",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 1297.21,
      "joules_per_output_token": 4.6699432,
      "median_itl_ms": 40.4,
      "p90_itl_ms": 41.1,
      "p99_itl_ms": 82.2,
      "tokens_per_second": 389.4,
      "gpu_power_watts_total": 1818,
      "avg_output_tokens": 871,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 41.5,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/16",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "FP8",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 32.0,
      "wh_per_million_output_tokens": 729.77,
      "joules_per_output_token": 2.6271713,
      "median_itl_ms": 43.2,
      "p90_itl_ms": 44.1,
      "p99_itl_ms": 90.6,
      "tokens_per_second": 721.2,
      "gpu_power_watts_total": 1895,
      "avg_output_tokens": 882,
      "under_itl_deadline": true,
      "avg_running_batch": 32.0,
      "p95_itl_ms": 44.5,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/32",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "FP8",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 408.89,
      "joules_per_output_token": 1.4719871,
      "median_itl_ms": 47.8,
      "p90_itl_ms": 49.5,
      "p99_itl_ms": 93.3,
      "tokens_per_second": 1285.4,
      "gpu_power_watts_total": 1892,
      "avg_output_tokens": 869,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 51.0,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/64",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "FP8",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.9,
      "wh_per_million_output_tokens": 228.26,
      "joules_per_output_token": 0.8217263,
      "median_itl_ms": 53.9,
      "p90_itl_ms": 57.9,
      "p99_itl_ms": 108.1,
      "tokens_per_second": 2255.5,
      "gpu_power_watts_total": 1853,
      "avg_output_tokens": 885,
      "under_itl_deadline": true,
      "avg_running_batch": 127.9,
      "p95_itl_ms": 61.8,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/128",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "FP8",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.7,
      "wh_per_million_output_tokens": 139.08,
      "joules_per_output_token": 0.5006937,
      "median_itl_ms": 64.5,
      "p90_itl_ms": 70.1,
      "p99_itl_ms": 119.5,
      "tokens_per_second": 3776.4,
      "gpu_power_watts_total": 1891,
      "avg_output_tokens": 876,
      "under_itl_deadline": true,
      "avg_running_batch": 255.7,
      "p95_itl_ms": 80.8,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/256",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 4,
      "precision": "FP8",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 2630.94,
      "joules_per_output_token": 9.4713711,
      "median_itl_ms": 35.2,
      "p90_itl_ms": 36.4,
      "p99_itl_ms": 37.1,
      "tokens_per_second": 223.8,
      "gpu_power_watts_total": 2120,
      "avg_output_tokens": 855,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 36.6,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/4/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 4,
      "precision": "FP8",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 1459.17,
      "joules_per_output_token": 5.2530122,
      "median_itl_ms": 37.3,
      "p90_itl_ms": 37.9,
      "p99_itl_ms": 67.9,
      "tokens_per_second": 420.7,
      "gpu_power_watts_total": 2210,
      "avg_output_tokens": 871,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 38.4,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/4/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 4,
      "precision": "FP8",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 32.0,
      "wh_per_million_output_tokens": 835.94,
      "joules_per_output_token": 3.0093965,
      "median_itl_ms": 42.8,
      "p90_itl_ms": 43.4,
      "p99_itl_ms": 69.8,
      "tokens_per_second": 731.6,
      "gpu_power_watts_total": 2202,
      "avg_output_tokens": 889,
      "under_itl_deadline": true,
      "avg_running_batch": 32.0,
      "p95_itl_ms": 43.8,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/4/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 4,
      "precision": "FP8",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 483.71,
      "joules_per_output_token": 1.7413487,
      "median_itl_ms": 49.0,
      "p90_itl_ms": 49.9,
      "p99_itl_ms": 90.4,
      "tokens_per_second": 1262.3,
      "gpu_power_watts_total": 2198,
      "avg_output_tokens": 863,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 51.2,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/4/64",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 4,
      "precision": "FP8",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.9,
      "wh_per_million_output_tokens": 352.24,
      "joules_per_output_token": 1.2680548,
      "median_itl_ms": 52.7,
      "p90_itl_ms": 53.9,
      "p99_itl_ms": 100.5,
      "tokens_per_second": 1752.5,
      "gpu_power_watts_total": 2222,
      "avg_output_tokens": 863,
      "under_itl_deadline": true,
      "avg_running_batch": 95.9,
      "p95_itl_ms": 58.1,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/4/96",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 4,
      "precision": "FP8",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.9,
      "wh_per_million_output_tokens": 305.79,
      "joules_per_output_token": 1.1008288,
      "median_itl_ms": 53.1,
      "p90_itl_ms": 56.2,
      "p99_itl_ms": 105.0,
      "tokens_per_second": 2294.9,
      "gpu_power_watts_total": 2526,
      "avg_output_tokens": 863,
      "under_itl_deadline": true,
      "avg_running_batch": 127.9,
      "p95_itl_ms": 60.4,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/4/128",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 4,
      "precision": "FP8",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.8,
      "wh_per_million_output_tokens": 233.4,
      "joules_per_output_token": 0.8402416,
      "median_itl_ms": 60.3,
      "p90_itl_ms": 63.3,
      "p99_itl_ms": 119.8,
      "tokens_per_second": 3015.6,
      "gpu_power_watts_total": 2534,
      "avg_output_tokens": 864,
      "under_itl_deadline": true,
      "avg_running_batch": 191.8,
      "p95_itl_ms": 79.1,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/4/192",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 3163.76,
      "joules_per_output_token": 11.3895504,
      "median_itl_ms": 24.7,
      "p90_itl_ms": 25.0,
      "p99_itl_ms": 25.9,
      "tokens_per_second": 315.8,
      "gpu_power_watts_total": 3596,
      "avg_output_tokens": 877,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 25.1,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 1749.15,
      "joules_per_output_token": 6.2969448,
      "median_itl_ms": 26.8,
      "p90_itl_ms": 27.2,
      "p99_itl_ms": 82.4,
      "tokens_per_second": 577.8,
      "gpu_power_watts_total": 3638,
      "avg_output_tokens": 873,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 27.4,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 32.0,
      "wh_per_million_output_tokens": 1012.34,
      "joules_per_output_token": 3.6444221,
      "median_itl_ms": 30.2,
      "p90_itl_ms": 30.6,
      "p99_itl_ms": 81.9,
      "tokens_per_second": 1016.9,
      "gpu_power_watts_total": 3706,
      "avg_output_tokens": 872,
      "under_itl_deadline": true,
      "avg_running_batch": 32.0,
      "p95_itl_ms": 31.0,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 587.77,
      "joules_per_output_token": 2.1159703,
      "median_itl_ms": 34.7,
      "p90_itl_ms": 35.3,
      "p99_itl_ms": 83.9,
      "tokens_per_second": 1760.5,
      "gpu_power_watts_total": 3725,
      "avg_output_tokens": 876,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 36.4,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/64",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.9,
      "wh_per_million_output_tokens": 451.85,
      "joules_per_output_token": 1.6266693,
      "median_itl_ms": 38.7,
      "p90_itl_ms": 39.7,
      "p99_itl_ms": 89.9,
      "tokens_per_second": 2346.7,
      "gpu_power_watts_total": 3817,
      "avg_output_tokens": 873,
      "under_itl_deadline": true,
      "avg_running_batch": 95.9,
      "p95_itl_ms": 41.6,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/96",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.8,
      "wh_per_million_output_tokens": 378.93,
      "joules_per_output_token": 1.3641469,
      "median_itl_ms": 42.1,
      "p90_itl_ms": 44.6,
      "p99_itl_ms": 101.2,
      "tokens_per_second": 2840.0,
      "gpu_power_watts_total": 3874,
      "avg_output_tokens": 879,
      "under_itl_deadline": true,
      "avg_running_batch": 127.8,
      "p95_itl_ms": 48.3,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/128",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.8,
      "wh_per_million_output_tokens": 306.66,
      "joules_per_output_token": 1.1039672,
      "median_itl_ms": 49.6,
      "p90_itl_ms": 53.0,
      "p99_itl_ms": 116.0,
      "tokens_per_second": 3609.4,
      "gpu_power_watts_total": 3985,
      "avg_output_tokens": 887,
      "under_itl_deadline": true,
      "avg_running_batch": 191.8,
      "p95_itl_ms": 59.9,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/192",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.7,
      "wh_per_million_output_tokens": 263.98,
      "joules_per_output_token": 0.9503259,
      "median_itl_ms": 54.9,
      "p90_itl_ms": 60.0,
      "p99_itl_ms": 123.1,
      "tokens_per_second": 4321.7,
      "gpu_power_watts_total": 4107,
      "avg_output_tokens": 862,
      "under_itl_deadline": true,
      "avg_running_batch": 255.7,
      "p95_itl_ms": 67.8,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/256",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 384,
      "avg_requests_in_flight": 383.5,
      "wh_per_million_output_tokens": 229.43,
      "joules_per_output_token": 0.8259378,
      "median_itl_ms": 66.6,
      "p90_itl_ms": 75.7,
      "p99_itl_ms": 160.4,
      "tokens_per_second": 5023.2,
      "gpu_power_watts_total": 4149,
      "avg_output_tokens": 874,
      "under_itl_deadline": true,
      "avg_running_batch": 383.5,
      "p95_itl_ms": 103.0,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/384",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 511.2,
      "wh_per_million_output_tokens": 205.96,
      "joules_per_output_token": 0.7414699,
      "median_itl_ms": 75.8,
      "p90_itl_ms": 115.9,
      "p99_itl_ms": 197.4,
      "tokens_per_second": 5342.0,
      "gpu_power_watts_total": 3961,
      "avg_output_tokens": 881,
      "under_itl_deadline": true,
      "avg_running_batch": 511.2,
      "p95_itl_ms": 130.9,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/512",
      "repeat": 1
    }
  ],
  "best": {
    "B200": {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 2,
      "precision": "FP8",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.7,
      "wh_per_million_output_tokens": 139.08,
      "joules_per_output_token": 0.5006937,
      "median_itl_ms": 64.5,
      "p90_itl_ms": 70.1,
      "p99_itl_ms": 119.5,
      "tokens_per_second": 3776.4,
      "gpu_power_watts_total": 1891,
      "avg_output_tokens": 876,
      "under_itl_deadline": true,
      "avg_running_batch": 255.7,
      "p95_itl_ms": 80.8,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/256",
      "repeat": 1
    },
    "H100": {
      "gpu": "H100",
      "num_gpus": 8,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 8,
      "precision": "FP8",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 511.2,
      "wh_per_million_output_tokens": 205.96,
      "joules_per_output_token": 0.7414699,
      "median_itl_ms": 75.8,
      "p90_itl_ms": 115.9,
      "p99_itl_ms": 197.4,
      "tokens_per_second": 5342.0,
      "gpu_power_watts_total": 3961,
      "avg_output_tokens": 881,
      "under_itl_deadline": true,
      "avg_running_batch": 511.2,
      "p95_itl_ms": 130.9,
      "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/512",
      "repeat": 1
    }
  }
}