{
  "version": "1",
  "updated": "2026-09-27",
  "data_version": "2026-09-27",
  "unit": "Wh per million output tokens",
  "itl_deadline_ms": 500,
  "source": {
    "name": "ML.ENERGY Leaderboard",
    "publisher": "The ML.ENERGY Initiative (started at SymbioticLab, University of Michigan)",
    "url": "https://ml.energy/leaderboard/",
    "data_url": "https://raw.githubusercontent.com/ml-energy/leaderboard/master/public/data/tasks/lm-arena-chat.json",
    "version": "3.0 (released 2025-12-01); data compiled 2026-02-16",
    "released": "2025-12-01",
    "compiled": "2026-02-16",
    "compiled_evidence": "the leaderboard's public/data/index.json last_updated, and the last commit to the task files, both 2026-02-16",
    "retrieved": "2026-09-26",
    "licence": "Apache-2.0 per the ml-energy/benchmark-v3 dataset card",
    "licence_evidence": [
      "Hugging Face dataset ml-energy/benchmark-v3: tagged license:apache-2.0, ships a LICENSE file, and its dataset card's License section reads Apache-2.0",
      "The ML.ENERGY Leaderboard web app's footer says Apache 2.0",
      "The leaderboard repository (github.com/ml-energy/leaderboard) carries no licence file; its JSON files are reproduced here as published measurements until the ML.ENERGY team confirms that the dataset licence covers them"
    ],
    "workload": "Text Conversation (lm-arena-chat)",
    "method": "Energy per output token is steady-state GPU energy divided by the output tokens generated in the steady-state window (the benchmark's steady_state_energy_per_token), measured with Zeus under vLLM 0.11.1 on H100 80GB HBM3 and B200 servers. Energy per request is that figure multiplied by the average output length, not a separate measurement. Prompt processing is included.",
    "selection": "Each index row is the lowest-energy configuration for its model and GPU whose median inter-token latency does not exceed 500 ms, the leaderboard's own default ceiling; no measured configuration reaches it.",
    "cite": "Chung et al., The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization, NeurIPS 2025 Datasets and Benchmarks, arXiv 2505.06371",
    "cite_url": "https://arxiv.org/abs/2505.06371",
    "used_for": "index, per-model configurations, concurrency curves, measured GPU power",
    "configurations": "328 stable configurations: the leaderboard publishes the runs that passed its stability filter; every run, including the unstable ones, is in the gated Hugging Face dataset ml-energy/benchmark-v3"
  },
  "caveat": "GPU energy only, measured at steady state on a busy server (the most energy-efficient configuration under a latency limit). Host, network, cooling and idle time are not counted. Every figure is a floor; a lightly used server sits well above it.",
  "boundary": "gpu_only",
  "slug": "llama-3-1-8b-instruct",
  "name": "Llama 3.1 8B Instruct",
  "model_id": "meta-llama/Llama-3.1-8B-Instruct",
  "params_b": 8.0,
  "active_params_b": 8.0,
  "architecture": "Dense Transformer",
  "gpus": [
    "B200",
    "H100"
  ],
  "configurations": [
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 108.48,
      "joules_per_output_token": 0.3905239,
      "median_itl_ms": 4.6,
      "p90_itl_ms": 6.0,
      "p99_itl_ms": 14.8,
      "tokens_per_second": 1637.3,
      "gpu_power_watts_total": 639,
      "avg_output_tokens": 388,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 7.7,
      "row_id": "llama-3-1-8b-instruct/B200/1/8",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 15.9,
      "wh_per_million_output_tokens": 67.32,
      "joules_per_output_token": 0.2423461,
      "median_itl_ms": 5.1,
      "p90_itl_ms": 8.8,
      "p99_itl_ms": 18.0,
      "tokens_per_second": 2750.9,
      "gpu_power_watts_total": 667,
      "avg_output_tokens": 409,
      "under_itl_deadline": true,
      "avg_running_batch": 15.9,
      "p95_itl_ms": 12.2,
      "row_id": "llama-3-1-8b-instruct/B200/1/16",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.8,
      "wh_per_million_output_tokens": 41.96,
      "joules_per_output_token": 0.1510693,
      "median_itl_ms": 6.3,
      "p90_itl_ms": 14.9,
      "p99_itl_ms": 25.1,
      "tokens_per_second": 3809.9,
      "gpu_power_watts_total": 576,
      "avg_output_tokens": 398,
      "under_itl_deadline": true,
      "avg_running_batch": 31.8,
      "p95_itl_ms": 18.3,
      "row_id": "llama-3-1-8b-instruct/B200/1/32",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.8,
      "wh_per_million_output_tokens": 30.64,
      "joules_per_output_token": 0.1103027,
      "median_itl_ms": 9.0,
      "p90_itl_ms": 20.0,
      "p99_itl_ms": 34.2,
      "tokens_per_second": 5589.3,
      "gpu_power_watts_total": 617,
      "avg_output_tokens": 390,
      "under_itl_deadline": true,
      "avg_running_batch": 63.8,
      "p95_itl_ms": 24.2,
      "row_id": "llama-3-1-8b-instruct/B200/1/64",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.5,
      "wh_per_million_output_tokens": 27.31,
      "joules_per_output_token": 0.0983167,
      "median_itl_ms": 26.7,
      "p90_itl_ms": 42.8,
      "p99_itl_ms": 58.4,
      "tokens_per_second": 4570.2,
      "gpu_power_watts_total": 449,
      "avg_output_tokens": 396,
      "under_itl_deadline": true,
      "avg_running_batch": 127.5,
      "p95_itl_ms": 48.3,
      "row_id": "llama-3-1-8b-instruct/B200/1/128",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.2,
      "wh_per_million_output_tokens": 19.58,
      "joules_per_output_token": 0.0704841,
      "median_itl_ms": 31.7,
      "p90_itl_ms": 48.5,
      "p99_itl_ms": 65.5,
      "tokens_per_second": 7664.1,
      "gpu_power_watts_total": 540,
      "avg_output_tokens": 399,
      "under_itl_deadline": true,
      "avg_running_batch": 255.2,
      "p95_itl_ms": 54.2,
      "row_id": "llama-3-1-8b-instruct/B200/1/256",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 510.6,
      "wh_per_million_output_tokens": 16.73,
      "joules_per_output_token": 0.060235,
      "median_itl_ms": 50.2,
      "p90_itl_ms": 71.4,
      "p99_itl_ms": 93.6,
      "tokens_per_second": 9571.4,
      "gpu_power_watts_total": 577,
      "avg_output_tokens": 396,
      "under_itl_deadline": true,
      "avg_running_batch": 510.6,
      "p95_itl_ms": 77.4,
      "row_id": "llama-3-1-8b-instruct/B200/1/512",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 1021.3,
      "wh_per_million_output_tokens": 15.26,
      "joules_per_output_token": 0.0549226,
      "median_itl_ms": 87.1,
      "p90_itl_ms": 113.6,
      "p99_itl_ms": 153.5,
      "tokens_per_second": 10922.5,
      "gpu_power_watts_total": 600,
      "avg_output_tokens": 396,
      "under_itl_deadline": true,
      "avg_running_batch": 1021.3,
      "p95_itl_ms": 121.8,
      "row_id": "llama-3-1-8b-instruct/B200/1/1024",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 2048,
      "avg_requests_in_flight": 2042.9,
      "wh_per_million_output_tokens": 12.97,
      "joules_per_output_token": 0.0466958,
      "median_itl_ms": 124.9,
      "p90_itl_ms": 175.2,
      "p99_itl_ms": 287.2,
      "tokens_per_second": 14799.3,
      "gpu_power_watts_total": 691,
      "avg_output_tokens": 396,
      "under_itl_deadline": true,
      "avg_running_batch": 2042.9,
      "p95_itl_ms": 190.3,
      "row_id": "llama-3-1-8b-instruct/B200/1/2048",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 126.33,
      "joules_per_output_token": 0.4547983,
      "median_itl_ms": 7.3,
      "p90_itl_ms": 7.7,
      "p99_itl_ms": 10.9,
      "tokens_per_second": 1069.6,
      "gpu_power_watts_total": 486,
      "avg_output_tokens": 389,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 8.0,
      "row_id": "llama-3-1-8b-instruct/H100/1/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 72.98,
      "joules_per_output_token": 0.2627398,
      "median_itl_ms": 7.9,
      "p90_itl_ms": 8.6,
      "p99_itl_ms": 13.3,
      "tokens_per_second": 1953.6,
      "gpu_power_watts_total": 513,
      "avg_output_tokens": 404,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 9.2,
      "row_id": "llama-3-1-8b-instruct/H100/1/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.9,
      "wh_per_million_output_tokens": 45.85,
      "joules_per_output_token": 0.1650551,
      "median_itl_ms": 8.9,
      "p90_itl_ms": 10.4,
      "p99_itl_ms": 18.4,
      "tokens_per_second": 3391.1,
      "gpu_power_watts_total": 560,
      "avg_output_tokens": 390,
      "under_itl_deadline": true,
      "avg_running_batch": 31.9,
      "p95_itl_ms": 11.4,
      "row_id": "llama-3-1-8b-instruct/H100/1/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.9,
      "wh_per_million_output_tokens": 32.94,
      "joules_per_output_token": 0.1185796,
      "median_itl_ms": 11.5,
      "p90_itl_ms": 13.2,
      "p99_itl_ms": 25.7,
      "tokens_per_second": 5147.2,
      "gpu_power_watts_total": 610,
      "avg_output_tokens": 396,
      "under_itl_deadline": true,
      "avg_running_batch": 63.9,
      "p95_itl_ms": 14.8,
      "row_id": "llama-3-1-8b-instruct/H100/1/64",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.7,
      "wh_per_million_output_tokens": 28.57,
      "joules_per_output_token": 0.1028655,
      "median_itl_ms": 14.1,
      "p90_itl_ms": 16.7,
      "p99_itl_ms": 35.4,
      "tokens_per_second": 6260.1,
      "gpu_power_watts_total": 644,
      "avg_output_tokens": 393,
      "under_itl_deadline": true,
      "avg_running_batch": 95.7,
      "p95_itl_ms": 20.1,
      "row_id": "llama-3-1-8b-instruct/H100/1/96",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.6,
      "wh_per_million_output_tokens": 26.42,
      "joules_per_output_token": 0.0951099,
      "median_itl_ms": 16.5,
      "p90_itl_ms": 20.0,
      "p99_itl_ms": 46.4,
      "tokens_per_second": 7036.5,
      "gpu_power_watts_total": 669,
      "avg_output_tokens": 405,
      "under_itl_deadline": true,
      "avg_running_batch": 127.6,
      "p95_itl_ms": 24.7,
      "row_id": "llama-3-1-8b-instruct/H100/1/128",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.7,
      "wh_per_million_output_tokens": 23.25,
      "joules_per_output_token": 0.0836952,
      "median_itl_ms": 20.9,
      "p90_itl_ms": 27.0,
      "p99_itl_ms": 59.0,
      "tokens_per_second": 8223.3,
      "gpu_power_watts_total": 688,
      "avg_output_tokens": 397,
      "under_itl_deadline": true,
      "avg_running_batch": 191.7,
      "p95_itl_ms": 33.5,
      "row_id": "llama-3-1-8b-instruct/H100/1/192",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.2,
      "wh_per_million_output_tokens": 21.29,
      "joules_per_output_token": 0.0766413,
      "median_itl_ms": 26.1,
      "p90_itl_ms": 32.5,
      "p99_itl_ms": 59.7,
      "tokens_per_second": 9009.6,
      "gpu_power_watts_total": 691,
      "avg_output_tokens": 404,
      "under_itl_deadline": true,
      "avg_running_batch": 255.2,
      "p95_itl_ms": 38.8,
      "row_id": "llama-3-1-8b-instruct/H100/1/256",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 384,
      "avg_requests_in_flight": 383.0,
      "wh_per_million_output_tokens": 20.62,
      "joules_per_output_token": 0.0742342,
      "median_itl_ms": 37.7,
      "p90_itl_ms": 47.8,
      "p99_itl_ms": 96.2,
      "tokens_per_second": 9248.0,
      "gpu_power_watts_total": 687,
      "avg_output_tokens": 394,
      "under_itl_deadline": true,
      "avg_running_batch": 383.0,
      "p95_itl_ms": 54.3,
      "row_id": "llama-3-1-8b-instruct/H100/1/384",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 510.9,
      "wh_per_million_output_tokens": 19.5,
      "joules_per_output_token": 0.0701945,
      "median_itl_ms": 47.4,
      "p90_itl_ms": 59.1,
      "p99_itl_ms": 106.8,
      "tokens_per_second": 9805.6,
      "gpu_power_watts_total": 688,
      "avg_output_tokens": 397,
      "under_itl_deadline": true,
      "avg_running_batch": 510.9,
      "p95_itl_ms": 65.0,
      "row_id": "llama-3-1-8b-instruct/H100/1/512",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 768,
      "avg_requests_in_flight": 690.9,
      "wh_per_million_output_tokens": 19.82,
      "joules_per_output_token": 0.0713482,
      "median_itl_ms": 62.9,
      "p90_itl_ms": 87.8,
      "p99_itl_ms": 192.0,
      "tokens_per_second": 9602.8,
      "gpu_power_watts_total": 685,
      "avg_output_tokens": 397,
      "under_itl_deadline": true,
      "avg_running_batch": 690.9,
      "p95_itl_ms": 107.0,
      "row_id": "llama-3-1-8b-instruct/H100/1/768",
      "repeat": 1
    }
  ],
  "best": {
    "B200": {
      "gpu": "B200",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 2048,
      "avg_requests_in_flight": 2042.9,
      "wh_per_million_output_tokens": 12.97,
      "joules_per_output_token": 0.0466958,
      "median_itl_ms": 124.9,
      "p90_itl_ms": 175.2,
      "p99_itl_ms": 287.2,
      "tokens_per_second": 14799.3,
      "gpu_power_watts_total": 691,
      "avg_output_tokens": 396,
      "under_itl_deadline": true,
      "avg_running_batch": 2042.9,
      "p95_itl_ms": 190.3,
      "row_id": "llama-3-1-8b-instruct/B200/1/2048",
      "repeat": 1
    },
    "H100": {
      "gpu": "H100",
      "num_gpus": 1,
      "tensor_parallel": 1,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 510.9,
      "wh_per_million_output_tokens": 19.5,
      "joules_per_output_token": 0.0701945,
      "median_itl_ms": 47.4,
      "p90_itl_ms": 59.1,
      "p99_itl_ms": 106.8,
      "tokens_per_second": 9805.6,
      "gpu_power_watts_total": 688,
      "avg_output_tokens": 397,
      "under_itl_deadline": true,
      "avg_running_batch": 510.9,
      "p95_itl_ms": 65.0,
      "row_id": "llama-3-1-8b-instruct/H100/1/512",
      "repeat": 1
    }
  }
}