{
  "version": "1",
  "updated": "2026-09-27",
  "data_version": "2026-09-27",
  "unit": "Wh per million output tokens",
  "itl_deadline_ms": 500,
  "source": {
    "name": "ML.ENERGY Leaderboard",
    "publisher": "The ML.ENERGY Initiative (started at SymbioticLab, University of Michigan)",
    "url": "https://ml.energy/leaderboard/",
    "data_url": "https://raw.githubusercontent.com/ml-energy/leaderboard/master/public/data/tasks/lm-arena-chat.json",
    "version": "3.0 (released 2025-12-01); data compiled 2026-02-16",
    "released": "2025-12-01",
    "compiled": "2026-02-16",
    "compiled_evidence": "the leaderboard's public/data/index.json last_updated, and the last commit to the task files, both 2026-02-16",
    "retrieved": "2026-09-26",
    "licence": "Apache-2.0 per the ml-energy/benchmark-v3 dataset card",
    "licence_evidence": [
      "Hugging Face dataset ml-energy/benchmark-v3: tagged license:apache-2.0, ships a LICENSE file, and its dataset card's License section reads Apache-2.0",
      "The ML.ENERGY Leaderboard web app's footer says Apache 2.0",
      "The leaderboard repository (github.com/ml-energy/leaderboard) carries no licence file; its JSON files are reproduced here as published measurements until the ML.ENERGY team confirms that the dataset licence covers them"
    ],
    "workload": "Text Conversation (lm-arena-chat)",
    "method": "Energy per output token is steady-state GPU energy divided by the output tokens generated in the steady-state window (the benchmark's steady_state_energy_per_token), measured with Zeus under vLLM 0.11.1 on H100 80GB HBM3 and B200 servers. Energy per request is that figure multiplied by the average output length, not a separate measurement. Prompt processing is included.",
    "selection": "Each index row is the lowest-energy configuration for its model and GPU whose median inter-token latency does not exceed 500 ms, the leaderboard's own default ceiling; no measured configuration reaches it.",
    "cite": "Chung et al., The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization, NeurIPS 2025 Datasets and Benchmarks, arXiv 2505.06371",
    "cite_url": "https://arxiv.org/abs/2505.06371",
    "used_for": "index, per-model configurations, concurrency curves, measured GPU power",
    "configurations": "328 stable configurations: the leaderboard publishes the runs that passed its stability filter; every run, including the unstable ones, is in the gated Hugging Face dataset ml-energy/benchmark-v3"
  },
  "caveat": "GPU energy only, measured at steady state on a busy server (the most energy-efficient configuration under a latency limit). Host, network, cooling and idle time are not counted. Every figure is a floor; a lightly used server sits well above it.",
  "boundary": "gpu_only",
  "slug": "llama-3-1-70b-instruct",
  "name": "Llama 3.1 70B Instruct",
  "model_id": "meta-llama/Llama-3.1-70B-Instruct",
  "params_b": 70.0,
  "active_params_b": 70.0,
  "architecture": "Dense Transformer",
  "gpus": [
    "B200",
    "H100"
  ],
  "configurations": [
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 991.46,
      "joules_per_output_token": 3.569266,
      "median_itl_ms": 16.4,
      "p90_itl_ms": 16.9,
      "p99_itl_ms": 22.2,
      "tokens_per_second": 475.7,
      "gpu_power_watts_total": 1698,
      "avg_output_tokens": 352,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 17.1,
      "row_id": "llama-3-1-70b-instruct/B200/2/8",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 15.9,
      "wh_per_million_output_tokens": 528.52,
      "joules_per_output_token": 1.9026558,
      "median_itl_ms": 16.8,
      "p90_itl_ms": 17.6,
      "p99_itl_ms": 31.3,
      "tokens_per_second": 910.8,
      "gpu_power_watts_total": 1733,
      "avg_output_tokens": 360,
      "under_itl_deadline": true,
      "avg_running_batch": 15.9,
      "p95_itl_ms": 18.3,
      "row_id": "llama-3-1-70b-instruct/B200/2/16",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.9,
      "wh_per_million_output_tokens": 303.2,
      "joules_per_output_token": 1.0915058,
      "median_itl_ms": 18.7,
      "p90_itl_ms": 20.3,
      "p99_itl_ms": 49.4,
      "tokens_per_second": 1589.6,
      "gpu_power_watts_total": 1735,
      "avg_output_tokens": 346,
      "under_itl_deadline": true,
      "avg_running_batch": 31.9,
      "p95_itl_ms": 23.9,
      "row_id": "llama-3-1-70b-instruct/B200/2/32",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.8,
      "wh_per_million_output_tokens": 193.63,
      "joules_per_output_token": 0.6970763,
      "median_itl_ms": 21.7,
      "p90_itl_ms": 25.5,
      "p99_itl_ms": 80.1,
      "tokens_per_second": 2597.8,
      "gpu_power_watts_total": 1811,
      "avg_output_tokens": 346,
      "under_itl_deadline": true,
      "avg_running_batch": 63.8,
      "p95_itl_ms": 34.1,
      "row_id": "llama-3-1-70b-instruct/B200/2/64",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.7,
      "wh_per_million_output_tokens": 133.6,
      "joules_per_output_token": 0.4809738,
      "median_itl_ms": 27.3,
      "p90_itl_ms": 35.8,
      "p99_itl_ms": 103.3,
      "tokens_per_second": 3977.5,
      "gpu_power_watts_total": 1913,
      "avg_output_tokens": 353,
      "under_itl_deadline": true,
      "avg_running_batch": 127.7,
      "p95_itl_ms": 50.1,
      "row_id": "llama-3-1-70b-instruct/B200/2/128",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.3,
      "wh_per_million_output_tokens": 100.4,
      "joules_per_output_token": 0.3614289,
      "median_itl_ms": 35.8,
      "p90_itl_ms": 60.0,
      "p99_itl_ms": 179.9,
      "tokens_per_second": 5372.4,
      "gpu_power_watts_total": 1942,
      "avg_output_tokens": 361,
      "under_itl_deadline": true,
      "avg_running_batch": 255.3,
      "p95_itl_ms": 82.1,
      "row_id": "llama-3-1-70b-instruct/B200/2/256",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 510.5,
      "wh_per_million_output_tokens": 75.22,
      "joules_per_output_token": 0.2707841,
      "median_itl_ms": 63.7,
      "p90_itl_ms": 83.2,
      "p99_itl_ms": 225.5,
      "tokens_per_second": 7134.0,
      "gpu_power_watts_total": 1932,
      "avg_output_tokens": 352,
      "under_itl_deadline": true,
      "avg_running_batch": 510.5,
      "p95_itl_ms": 103.7,
      "row_id": "llama-3-1-70b-instruct/B200/2/512",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 1021.5,
      "wh_per_million_output_tokens": 62.97,
      "joules_per_output_token": 0.2266746,
      "median_itl_ms": 105.4,
      "p90_itl_ms": 128.1,
      "p99_itl_ms": 516.3,
      "tokens_per_second": 8551.4,
      "gpu_power_watts_total": 1938,
      "avg_output_tokens": 351,
      "under_itl_deadline": true,
      "avg_running_batch": 1021.5,
      "p95_itl_ms": 153.8,
      "row_id": "llama-3-1-70b-instruct/B200/2/1024",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1536,
      "avg_requests_in_flight": 1407.3,
      "wh_per_million_output_tokens": 61.39,
      "joules_per_output_token": 0.2209983,
      "median_itl_ms": 134.8,
      "p90_itl_ms": 184.4,
      "p99_itl_ms": 536.4,
      "tokens_per_second": 8682.2,
      "gpu_power_watts_total": 1919,
      "avg_output_tokens": 350,
      "under_itl_deadline": true,
      "avg_running_batch": 1407.3,
      "p95_itl_ms": 224.8,
      "row_id": "llama-3-1-70b-instruct/B200/2/1536",
      "repeat": 1
    },
    {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 2048,
      "avg_requests_in_flight": 2041.9,
      "wh_per_million_output_tokens": 52.41,
      "joules_per_output_token": 0.1886745,
      "median_itl_ms": 140.5,
      "p90_itl_ms": 211.7,
      "p99_itl_ms": 558.6,
      "tokens_per_second": 10107.8,
      "gpu_power_watts_total": 1907,
      "avg_output_tokens": 351,
      "under_itl_deadline": true,
      "avg_running_batch": 2041.9,
      "p95_itl_ms": 251.8,
      "row_id": "llama-3-1-70b-instruct/B200/2/2048",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 8,
      "avg_requests_in_flight": 8.0,
      "wh_per_million_output_tokens": 1044.09,
      "joules_per_output_token": 3.7587139,
      "median_itl_ms": 16.6,
      "p90_itl_ms": 16.9,
      "p99_itl_ms": 21.1,
      "tokens_per_second": 469.7,
      "gpu_power_watts_total": 1765,
      "avg_output_tokens": 350,
      "under_itl_deadline": true,
      "avg_running_batch": 8.0,
      "p95_itl_ms": 17.0,
      "row_id": "llama-3-1-70b-instruct/H100/4/8",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 16,
      "avg_requests_in_flight": 16.0,
      "wh_per_million_output_tokens": 580.84,
      "joules_per_output_token": 2.0910102,
      "median_itl_ms": 17.7,
      "p90_itl_ms": 18.1,
      "p99_itl_ms": 32.1,
      "tokens_per_second": 862.6,
      "gpu_power_watts_total": 1804,
      "avg_output_tokens": 354,
      "under_itl_deadline": true,
      "avg_running_batch": 16.0,
      "p95_itl_ms": 18.4,
      "row_id": "llama-3-1-70b-instruct/H100/4/16",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 32,
      "avg_requests_in_flight": 31.9,
      "wh_per_million_output_tokens": 335.24,
      "joules_per_output_token": 1.2068752,
      "median_itl_ms": 20.0,
      "p90_itl_ms": 20.7,
      "p99_itl_ms": 56.3,
      "tokens_per_second": 1486.7,
      "gpu_power_watts_total": 1794,
      "avg_output_tokens": 351,
      "under_itl_deadline": true,
      "avg_running_batch": 31.9,
      "p95_itl_ms": 23.1,
      "row_id": "llama-3-1-70b-instruct/H100/4/32",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 64,
      "avg_requests_in_flight": 63.8,
      "wh_per_million_output_tokens": 221.46,
      "joules_per_output_token": 0.7972486,
      "median_itl_ms": 22.4,
      "p90_itl_ms": 24.4,
      "p99_itl_ms": 90.5,
      "tokens_per_second": 2510.5,
      "gpu_power_watts_total": 2002,
      "avg_output_tokens": 352,
      "under_itl_deadline": true,
      "avg_running_batch": 63.8,
      "p95_itl_ms": 32.0,
      "row_id": "llama-3-1-70b-instruct/H100/4/64",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 96,
      "avg_requests_in_flight": 95.7,
      "wh_per_million_output_tokens": 179.93,
      "joules_per_output_token": 0.6477576,
      "median_itl_ms": 25.4,
      "p90_itl_ms": 28.4,
      "p99_itl_ms": 114.1,
      "tokens_per_second": 3244.3,
      "gpu_power_watts_total": 2102,
      "avg_output_tokens": 357,
      "under_itl_deadline": true,
      "avg_running_batch": 95.7,
      "p95_itl_ms": 44.5,
      "row_id": "llama-3-1-70b-instruct/H100/4/96",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 128,
      "avg_requests_in_flight": 127.6,
      "wh_per_million_output_tokens": 156.49,
      "joules_per_output_token": 0.5633656,
      "median_itl_ms": 27.9,
      "p90_itl_ms": 34.0,
      "p99_itl_ms": 121.3,
      "tokens_per_second": 3876.3,
      "gpu_power_watts_total": 2184,
      "avg_output_tokens": 359,
      "under_itl_deadline": true,
      "avg_running_batch": 127.6,
      "p95_itl_ms": 54.7,
      "row_id": "llama-3-1-70b-instruct/H100/4/128",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 192,
      "avg_requests_in_flight": 191.6,
      "wh_per_million_output_tokens": 142.78,
      "joules_per_output_token": 0.5140014,
      "median_itl_ms": 33.7,
      "p90_itl_ms": 56.6,
      "p99_itl_ms": 161.5,
      "tokens_per_second": 4461.2,
      "gpu_power_watts_total": 2293,
      "avg_output_tokens": 357,
      "under_itl_deadline": true,
      "avg_running_batch": 191.6,
      "p95_itl_ms": 77.6,
      "row_id": "llama-3-1-70b-instruct/H100/4/192",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 256,
      "avg_requests_in_flight": 255.3,
      "wh_per_million_output_tokens": 133.05,
      "joules_per_output_token": 0.4789926,
      "median_itl_ms": 39.8,
      "p90_itl_ms": 68.5,
      "p99_itl_ms": 240.4,
      "tokens_per_second": 4915.1,
      "gpu_power_watts_total": 2354,
      "avg_output_tokens": 349,
      "under_itl_deadline": true,
      "avg_running_batch": 255.3,
      "p95_itl_ms": 92.9,
      "row_id": "llama-3-1-70b-instruct/H100/4/256",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 384,
      "avg_requests_in_flight": 383.0,
      "wh_per_million_output_tokens": 122.39,
      "joules_per_output_token": 0.440615,
      "median_itl_ms": 55.1,
      "p90_itl_ms": 92.4,
      "p99_itl_ms": 436.5,
      "tokens_per_second": 5318.8,
      "gpu_power_watts_total": 2344,
      "avg_output_tokens": 364,
      "under_itl_deadline": true,
      "avg_running_batch": 383.0,
      "p95_itl_ms": 114.9,
      "row_id": "llama-3-1-70b-instruct/H100/4/384",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 512,
      "avg_requests_in_flight": 510.5,
      "wh_per_million_output_tokens": 116.95,
      "joules_per_output_token": 0.42101,
      "median_itl_ms": 73.9,
      "p90_itl_ms": 103.7,
      "p99_itl_ms": 299.1,
      "tokens_per_second": 5921.3,
      "gpu_power_watts_total": 2493,
      "avg_output_tokens": 350,
      "under_itl_deadline": true,
      "avg_running_batch": 510.5,
      "p95_itl_ms": 136.4,
      "row_id": "llama-3-1-70b-instruct/H100/4/512",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 768,
      "avg_requests_in_flight": 766.3,
      "wh_per_million_output_tokens": 108.22,
      "joules_per_output_token": 0.389603,
      "median_itl_ms": 103.6,
      "p90_itl_ms": 135.2,
      "p99_itl_ms": 561.7,
      "tokens_per_second": 6507.5,
      "gpu_power_watts_total": 2535,
      "avg_output_tokens": 351,
      "under_itl_deadline": true,
      "avg_running_batch": 766.3,
      "p95_itl_ms": 152.3,
      "row_id": "llama-3-1-70b-instruct/H100/4/768",
      "repeat": 1
    },
    {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 898.5,
      "wh_per_million_output_tokens": 103.65,
      "joules_per_output_token": 0.373142,
      "median_itl_ms": 105.6,
      "p90_itl_ms": 151.7,
      "p99_itl_ms": 592.9,
      "tokens_per_second": 6827.2,
      "gpu_power_watts_total": 2548,
      "avg_output_tokens": 351,
      "under_itl_deadline": true,
      "avg_running_batch": 898.5,
      "p95_itl_ms": 219.9,
      "row_id": "llama-3-1-70b-instruct/H100/4/1024",
      "repeat": 1
    }
  ],
  "best": {
    "B200": {
      "gpu": "B200",
      "num_gpus": 2,
      "tensor_parallel": 2,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 2048,
      "avg_requests_in_flight": 2041.9,
      "wh_per_million_output_tokens": 52.41,
      "joules_per_output_token": 0.1886745,
      "median_itl_ms": 140.5,
      "p90_itl_ms": 211.7,
      "p99_itl_ms": 558.6,
      "tokens_per_second": 10107.8,
      "gpu_power_watts_total": 1907,
      "avg_output_tokens": 351,
      "under_itl_deadline": true,
      "avg_running_batch": 2041.9,
      "p95_itl_ms": 251.8,
      "row_id": "llama-3-1-70b-instruct/B200/2/2048",
      "repeat": 1
    },
    "H100": {
      "gpu": "H100",
      "num_gpus": 4,
      "tensor_parallel": 4,
      "data_parallel": 1,
      "expert_parallel": 1,
      "precision": "BF16",
      "max_num_seqs": 1024,
      "avg_requests_in_flight": 898.5,
      "wh_per_million_output_tokens": 103.65,
      "joules_per_output_token": 0.373142,
      "median_itl_ms": 105.6,
      "p90_itl_ms": 151.7,
      "p99_itl_ms": 592.9,
      "tokens_per_second": 6827.2,
      "gpu_power_watts_total": 2548,
      "avg_output_tokens": 351,
      "under_itl_deadline": true,
      "avg_running_batch": 898.5,
      "p95_itl_ms": 219.9,
      "row_id": "llama-3-1-70b-instruct/H100/4/1024",
      "repeat": 1
    }
  }
}