{ "version": "1", "updated": "2026-09-27", "data_version": "2026-09-27", "unit": "Wh per million output tokens", "workload": "text-conversation", "itl_deadline_ms": 500, "selection": "per model and GPU, the configuration with the lowest energy per token whose median inter-token latency does not exceed itl_deadline_ms (the leaderboard's default ceiling; no measured configuration reaches it)", "reference_row": { "model": "Qwen 3 32B", "gpu": "H100", "num_gpus": 2 }, "source": { "name": "ML.ENERGY Leaderboard", "publisher": "The ML.ENERGY Initiative (started at SymbioticLab, University of Michigan)", "url": "https://ml.energy/leaderboard/", "data_url": "https://raw.githubusercontent.com/ml-energy/leaderboard/master/public/data/tasks/lm-arena-chat.json", "version": "3.0 (released 2025-12-01); data compiled 2026-02-16", "released": "2025-12-01", "compiled": "2026-02-16", "compiled_evidence": "the leaderboard's public/data/index.json last_updated, and the last commit to the task files, both 2026-02-16", "retrieved": "2026-09-26", "licence": "Apache-2.0 per the ml-energy/benchmark-v3 dataset card", "licence_evidence": [ "Hugging Face dataset ml-energy/benchmark-v3: tagged license:apache-2.0, ships a LICENSE file, and its dataset card's License section reads Apache-2.0", "The ML.ENERGY Leaderboard web app's footer says Apache 2.0", "The leaderboard repository (github.com/ml-energy/leaderboard) carries no licence file; its JSON files are reproduced here as published measurements until the ML.ENERGY team confirms that the dataset licence covers them" ], "workload": "Text Conversation (lm-arena-chat)", "method": "Energy per output token is steady-state GPU energy divided by the output tokens generated in the steady-state window (the benchmark's steady_state_energy_per_token), measured with Zeus under vLLM 0.11.1 on H100 80GB HBM3 and B200 servers. Energy per request is that figure multiplied by the average output length, not a separate measurement. Prompt processing is included.", "selection": "Each index row is the lowest-energy configuration for its model and GPU whose median inter-token latency does not exceed 500 ms, the leaderboard's own default ceiling; no measured configuration reaches it.", "cite": "Chung et al., The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization, NeurIPS 2025 Datasets and Benchmarks, arXiv 2505.06371", "cite_url": "https://arxiv.org/abs/2505.06371", "used_for": "index, per-model configurations, concurrency curves, measured GPU power", "configurations": "328 stable configurations: the leaderboard publishes the runs that passed its stability filter; every run, including the unstable ones, is in the gated Hugging Face dataset ml-energy/benchmark-v3" }, "caveat": "GPU energy only, measured at steady state on a busy server (the most energy-efficient configuration under a latency limit). Host, network, cooling and idle time are not counted. Every figure is a floor; a lightly used server sits well above it.", "boundary": "gpu_only", "rows": [ { "model": "Llama 3.1 8B Instruct", "slug": "llama-3-1-8b-instruct", "model_id": "meta-llama/Llama-3.1-8B-Instruct", "params_b": 8.0, "active_params_b": 8.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 12.97, "joules_per_output_token": 0.0466958, "median_itl_ms": 124.9, "avg_requests_in_flight": 2042.9, "gpu_power_watts_total": 691, "avg_output_tokens": 396, "avg_running_batch": 2042.9, "max_num_seqs": 2048, "row_id": "llama-3-1-8b-instruct/B200/1/2048" }, { "model": "Qwen 3 8B", "slug": "qwen-3-8b", "model_id": "Qwen/Qwen3-8B", "params_b": 8.0, "active_params_b": 8.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 14.74, "joules_per_output_token": 0.0530581, "median_itl_ms": 91.2, "avg_requests_in_flight": 1426.7, "gpu_power_watts_total": 801, "avg_output_tokens": 649, "avg_running_batch": 1426.7, "max_num_seqs": 1536, "row_id": "qwen-3-8b/B200/1/1536" }, { "model": "Qwen 3 14B", "slug": "qwen-3-14b", "model_id": "Qwen/Qwen3-14B", "params_b": 14.0, "active_params_b": 14.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 15.21, "joules_per_output_token": 0.0547711, "median_itl_ms": 112.6, "avg_requests_in_flight": 2044.6, "gpu_power_watts_total": 869, "avg_output_tokens": 613, "avg_running_batch": 2044.6, "max_num_seqs": 2048, "row_id": "qwen-3-14b/B200/1/2048" }, { "model": "Qwen 3 30B A3B Instruct", "slug": "qwen-3-30b-a3b-instruct", "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "params_b": 30.0, "active_params_b": 3.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 15.67, "joules_per_output_token": 0.0564083, "median_itl_ms": 88.7, "avg_requests_in_flight": 1022.6, "gpu_power_watts_total": 601, "avg_output_tokens": 928, "avg_running_batch": 1022.6, "max_num_seqs": 1024, "row_id": "qwen-3-30b-a3b-instruct/B200/1/1024" }, { "model": "Llama 3.1 8B Instruct", "slug": "llama-3-1-8b-instruct", "model_id": "meta-llama/Llama-3.1-8B-Instruct", "params_b": 8.0, "active_params_b": 8.0, "precision": "BF16", "gpu": "H100", "num_gpus": 1, "wh_per_million_output_tokens": 19.5, "joules_per_output_token": 0.0701945, "median_itl_ms": 47.4, "avg_requests_in_flight": 510.9, "gpu_power_watts_total": 688, "avg_output_tokens": 397, "avg_running_batch": 510.9, "max_num_seqs": 512, "row_id": "llama-3-1-8b-instruct/H100/1/512" }, { "model": "Qwen 3 8B", "slug": "qwen-3-8b", "model_id": "Qwen/Qwen3-8B", "params_b": 8.0, "active_params_b": 8.0, "precision": "BF16", "gpu": "H100", "num_gpus": 1, "wh_per_million_output_tokens": 20.04, "joules_per_output_token": 0.0721292, "median_itl_ms": 47.2, "avg_requests_in_flight": 474.4, "gpu_power_watts_total": 616, "avg_output_tokens": 645, "avg_running_batch": 474.4, "max_num_seqs": 512, "row_id": "qwen-3-8b/H100/1/512" }, { "model": "Qwen 3 30B A3B Instruct", "slug": "qwen-3-30b-a3b-instruct", "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "params_b": 30.0, "active_params_b": 3.0, "precision": "BF16", "gpu": "H100", "num_gpus": 2, "wh_per_million_output_tokens": 24.69, "joules_per_output_token": 0.0888773, "median_itl_ms": 74.8, "avg_requests_in_flight": 1018.8, "gpu_power_watts_total": 1123, "avg_output_tokens": 920, "avg_running_batch": 1018.8, "max_num_seqs": 1024, "row_id": "qwen-3-30b-a3b-instruct/H100/2/1024" }, { "model": "Gemma 3 12B", "slug": "gemma-3-12b", "model_id": "google/gemma-3-12b-it", "params_b": 12.0, "active_params_b": 12.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 26.49, "joules_per_output_token": 0.0953556, "median_itl_ms": 43.5, "avg_requests_in_flight": 486.6, "gpu_power_watts_total": 928, "avg_output_tokens": 723, "avg_running_batch": 486.6, "max_num_seqs": 512, "row_id": "gemma-3-12b/B200/1/512" }, { "model": "Qwen 3 14B", "slug": "qwen-3-14b", "model_id": "Qwen/Qwen3-14B", "params_b": 14.0, "active_params_b": 14.0, "precision": "BF16", "gpu": "H100", "num_gpus": 1, "wh_per_million_output_tokens": 31.09, "joules_per_output_token": 0.1119169, "median_itl_ms": 38.4, "avg_requests_in_flight": 253.4, "gpu_power_watts_total": 645, "avg_output_tokens": 625, "avg_running_batch": 253.4, "max_num_seqs": 256, "row_id": "qwen-3-14b/H100/1/256" }, { "model": "Qwen 3 32B", "slug": "qwen-3-32b", "model_id": "Qwen/Qwen3-32B", "params_b": 32.0, "active_params_b": 32.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 39.05, "joules_per_output_token": 0.1405812, "median_itl_ms": 50.2, "avg_requests_in_flight": 383.3, "gpu_power_watts_total": 972, "avg_output_tokens": 635, "avg_running_batch": 383.3, "max_num_seqs": 384, "row_id": "qwen-3-32b/B200/1/384" }, { "model": "NVIDIA Nemotron Nano 12B V2", "slug": "nvidia-nemotron-nano-12b-v2", "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "params_b": 12.0, "active_params_b": 12.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 39.43, "joules_per_output_token": 0.1419623, "median_itl_ms": 33.9, "avg_requests_in_flight": 255.6, "gpu_power_watts_total": 885, "avg_output_tokens": 1464, "avg_running_batch": 255.6, "max_num_seqs": 256, "row_id": "nvidia-nemotron-nano-12b-v2/B200/1/256" }, { "model": "NVIDIA Nemotron Nano 9B V2", "slug": "nvidia-nemotron-nano-9b-v2", "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "params_b": 9.0, "active_params_b": 9.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 40.94, "joules_per_output_token": 0.1473681, "median_itl_ms": 40.7, "avg_requests_in_flight": 255.3, "gpu_power_watts_total": 800, "avg_output_tokens": 1242, "avg_running_batch": 255.3, "max_num_seqs": 256, "row_id": "nvidia-nemotron-nano-9b-v2/B200/1/256" }, { "model": "NVIDIA Nemotron Nano 9B V2", "slug": "nvidia-nemotron-nano-9b-v2", "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "params_b": 9.0, "active_params_b": 9.0, "precision": "BF16", "gpu": "H100", "num_gpus": 1, "wh_per_million_output_tokens": 43.11, "joules_per_output_token": 0.1552074, "median_itl_ms": 43.4, "avg_requests_in_flight": 255.3, "gpu_power_watts_total": 693, "avg_output_tokens": 510, "avg_running_batch": 255.3, "max_num_seqs": 256, "row_id": "nvidia-nemotron-nano-9b-v2/H100/1/256" }, { "model": "Gemma 3 12B", "slug": "gemma-3-12b", "model_id": "google/gemma-3-12b-it", "params_b": 12.0, "active_params_b": 12.0, "precision": "BF16", "gpu": "H100", "num_gpus": 1, "wh_per_million_output_tokens": 46.14, "joules_per_output_token": 0.1661133, "median_itl_ms": 34.7, "avg_requests_in_flight": 127.8, "gpu_power_watts_total": 589, "avg_output_tokens": 723, "avg_running_batch": 127.8, "max_num_seqs": 128, "row_id": "gemma-3-12b/H100/1/128" }, { "model": "Gemma 3 27B", "slug": "gemma-3-27b", "model_id": "google/gemma-3-27b-it", "params_b": 27.0, "active_params_b": 27.0, "precision": "BF16", "gpu": "B200", "num_gpus": 1, "wh_per_million_output_tokens": 49.1, "joules_per_output_token": 0.1767455, "median_itl_ms": 39.6, "avg_requests_in_flight": 247.3, "gpu_power_watts_total": 974, "avg_output_tokens": 717, "avg_running_batch": 247.3, "max_num_seqs": 256, "row_id": "gemma-3-27b/B200/1/256" }, { "model": "NVIDIA Nemotron Nano 12B V2", "slug": "nvidia-nemotron-nano-12b-v2", "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "params_b": 12.0, "active_params_b": 12.0, "precision": "BF16", "gpu": "H100", "num_gpus": 1, "wh_per_million_output_tokens": 50.08, "joules_per_output_token": 0.1803029, "median_itl_ms": 46.0, "avg_requests_in_flight": 241.2, "gpu_power_watts_total": 694, "avg_output_tokens": 462, "avg_running_batch": 241.2, "max_num_seqs": 256, "row_id": "nvidia-nemotron-nano-12b-v2/H100/1/256" }, { "model": "Llama 3.1 70B Instruct", "slug": "llama-3-1-70b-instruct", "model_id": "meta-llama/Llama-3.1-70B-Instruct", "params_b": 70.0, "active_params_b": 70.0, "precision": "BF16", "gpu": "B200", "num_gpus": 2, "wh_per_million_output_tokens": 52.41, "joules_per_output_token": 0.1886745, "median_itl_ms": 140.5, "avg_requests_in_flight": 2041.9, "gpu_power_watts_total": 1907, "avg_output_tokens": 351, "avg_running_batch": 2041.9, "max_num_seqs": 2048, "row_id": "llama-3-1-70b-instruct/B200/2/2048" }, { "model": "Llama 3.3 70B Instruct", "slug": "llama-3-3-70b-instruct", "model_id": "meta-llama/Llama-3.3-70B-Instruct", "params_b": 70.0, "active_params_b": 70.0, "precision": "BF16", "gpu": "B200", "num_gpus": 2, "wh_per_million_output_tokens": 60.72, "joules_per_output_token": 0.218586, "median_itl_ms": 105.6, "avg_requests_in_flight": 1021.6, "gpu_power_watts_total": 1935, "avg_output_tokens": 409, "avg_running_batch": 1021.6, "max_num_seqs": 1024, "row_id": "llama-3-3-70b-instruct/B200/2/1024" }, { "model": "Qwen 3 32B", "slug": "qwen-3-32b", "model_id": "Qwen/Qwen3-32B", "params_b": 32.0, "active_params_b": 32.0, "precision": "BF16", "gpu": "H100", "num_gpus": 2, "wh_per_million_output_tokens": 63.47, "joules_per_output_token": 0.2284806, "median_itl_ms": 44.5, "avg_requests_in_flight": 255.6, "gpu_power_watts_total": 1178, "avg_output_tokens": 634, "avg_running_batch": 255.6, "max_num_seqs": 256, "row_id": "qwen-3-32b/H100/2/256" }, { "model": "Qwen 3 235B A22B Instruct", "slug": "qwen-3-235b-a22b-instruct", "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "params_b": 235.0, "active_params_b": 22.0, "precision": "BF16", "gpu": "B200", "num_gpus": 4, "wh_per_million_output_tokens": 66.73, "joules_per_output_token": 0.2402453, "median_itl_ms": 67.8, "avg_requests_in_flight": 3764.6, "gpu_power_watts_total": 3196, "avg_output_tokens": 874, "avg_running_batch": 3764.6, "max_num_seqs": 4096, "row_id": "qwen-3-235b-a22b-instruct/B200/4/4096" }, { "model": "Gemma 3 27B", "slug": "gemma-3-27b", "model_id": "google/gemma-3-27b-it", "params_b": 27.0, "active_params_b": 27.0, "precision": "BF16", "gpu": "H100", "num_gpus": 2, "wh_per_million_output_tokens": 80.42, "joules_per_output_token": 0.2895195, "median_itl_ms": 39.8, "avg_requests_in_flight": 187.0, "gpu_power_watts_total": 1210, "avg_output_tokens": 714, "avg_running_batch": 187.0, "max_num_seqs": 192, "row_id": "gemma-3-27b/H100/2/192" }, { "model": "Llama 4 Scout 17B 16E Instruct", "slug": "llama-4-scout-17b-16e-instruct", "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "params_b": 109.0, "active_params_b": 17.0, "precision": "BF16", "gpu": "H100", "num_gpus": 4, "wh_per_million_output_tokens": 93.39, "joules_per_output_token": 0.3361983, "median_itl_ms": 69.1, "avg_requests_in_flight": 472.8, "gpu_power_watts_total": 1881, "avg_output_tokens": 402, "avg_running_batch": 472.8, "max_num_seqs": 512, "row_id": "llama-4-scout-17b-16e-instruct/H100/4/512" }, { "model": "Llama 3.1 70B Instruct", "slug": "llama-3-1-70b-instruct", "model_id": "meta-llama/Llama-3.1-70B-Instruct", "params_b": 70.0, "active_params_b": 70.0, "precision": "BF16", "gpu": "H100", "num_gpus": 4, "wh_per_million_output_tokens": 103.65, "joules_per_output_token": 0.373142, "median_itl_ms": 105.6, "avg_requests_in_flight": 898.5, "gpu_power_watts_total": 2548, "avg_output_tokens": 351, "avg_running_batch": 898.5, "max_num_seqs": 1024, "row_id": "llama-3-1-70b-instruct/H100/4/1024" }, { "model": "Llama 3.3 70B Instruct", "slug": "llama-3-3-70b-instruct", "model_id": "meta-llama/Llama-3.3-70B-Instruct", "params_b": 70.0, "active_params_b": 70.0, "precision": "BF16", "gpu": "H100", "num_gpus": 4, "wh_per_million_output_tokens": 117.53, "joules_per_output_token": 0.4230942, "median_itl_ms": 97.5, "avg_requests_in_flight": 724.2, "gpu_power_watts_total": 2546, "avg_output_tokens": 413, "avg_running_batch": 724.2, "max_num_seqs": 768, "row_id": "llama-3-3-70b-instruct/H100/4/768" }, { "model": "Llama 4 Maverick 17B 128E Instruct FP8", "slug": "llama-4-maverick-17b-128e-instruct-fp8", "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "params_b": 400.0, "active_params_b": 17.0, "precision": "FP8", "gpu": "H100", "num_gpus": 8, "wh_per_million_output_tokens": 134.5, "joules_per_output_token": 0.4841928, "median_itl_ms": 47.9, "avg_requests_in_flight": 383.3, "gpu_power_watts_total": 3462, "avg_output_tokens": 408, "avg_running_batch": 383.3, "max_num_seqs": 384, "row_id": "llama-4-maverick-17b-128e-instruct-fp8/H100/8/384" }, { "model": "Qwen 3 235B A22B Instruct FP8", "slug": "qwen-3-235b-a22b-instruct-fp8", "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "params_b": 235.0, "active_params_b": 22.0, "precision": "FP8", "gpu": "B200", "num_gpus": 2, "wh_per_million_output_tokens": 139.08, "joules_per_output_token": 0.5006937, "median_itl_ms": 64.5, "avg_requests_in_flight": 255.7, "gpu_power_watts_total": 1891, "avg_output_tokens": 876, "avg_running_batch": 255.7, "max_num_seqs": 256, "row_id": "qwen-3-235b-a22b-instruct-fp8/B200/2/256" }, { "model": "DeepSeek V3.1", "slug": "deepseek-v3-1", "model_id": "deepseek-ai/DeepSeek-V3.1", "params_b": 671.0, "active_params_b": 37.0, "precision": "FP8", "gpu": "B200", "num_gpus": 8, "wh_per_million_output_tokens": 168.43, "joules_per_output_token": 0.6063443, "median_itl_ms": 185.5, "avg_requests_in_flight": 3007.5, "gpu_power_watts_total": 5470, "avg_output_tokens": 842, "avg_running_batch": 3007.5, "max_num_seqs": 3072, "row_id": "deepseek-v3-1/B200/8/3072" }, { "model": "Qwen 3 235B A22B Instruct FP8", "slug": "qwen-3-235b-a22b-instruct-fp8", "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "params_b": 235.0, "active_params_b": 22.0, "precision": "FP8", "gpu": "H100", "num_gpus": 8, "wh_per_million_output_tokens": 205.96, "joules_per_output_token": 0.7414699, "median_itl_ms": 75.8, "avg_requests_in_flight": 511.2, "gpu_power_watts_total": 3961, "avg_output_tokens": 881, "avg_running_batch": 511.2, "max_num_seqs": 512, "row_id": "qwen-3-235b-a22b-instruct-fp8/H100/8/512" }, { "model": "Llama 3.1 405B Instruct", "slug": "llama-3-1-405b-instruct", "model_id": "meta-llama/Llama-3.1-405B-Instruct", "params_b": 405.0, "active_params_b": 405.0, "precision": "BF16", "gpu": "B200", "num_gpus": 8, "wh_per_million_output_tokens": 281.96, "joules_per_output_token": 1.0150418, "median_itl_ms": 272.3, "avg_requests_in_flight": 3588.7, "gpu_power_watts_total": 7530, "avg_output_tokens": 370, "avg_running_batch": 3588.7, "max_num_seqs": 4096, "row_id": "llama-3-1-405b-instruct/B200/8/4096" }, { "model": "Qwen 3 235B A22B Instruct", "slug": "qwen-3-235b-a22b-instruct", "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "params_b": 235.0, "active_params_b": 22.0, "precision": "BF16", "gpu": "H100", "num_gpus": 8, "wh_per_million_output_tokens": 303.53, "joules_per_output_token": 1.0927101, "median_itl_ms": 55.5, "avg_requests_in_flight": 228.5, "gpu_power_watts_total": 4038, "avg_output_tokens": 874, "avg_running_batch": 228.5, "max_num_seqs": 256, "row_id": "qwen-3-235b-a22b-instruct/H100/8/256" }, { "model": "Llama 3.1 405B Instruct FP8", "slug": "llama-3-1-405b-instruct-fp8", "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "params_b": 405.0, "active_params_b": 405.0, "precision": "FP8", "gpu": "B200", "num_gpus": 4, "wh_per_million_output_tokens": 318.53, "joules_per_output_token": 1.1466946, "median_itl_ms": 98.2, "avg_requests_in_flight": 510.5, "gpu_power_watts_total": 3883, "avg_output_tokens": 380, "avg_running_batch": 510.5, "max_num_seqs": 512, "row_id": "llama-3-1-405b-instruct-fp8/B200/4/512" }, { "model": "Llama 3.1 405B Instruct FP8", "slug": "llama-3-1-405b-instruct-fp8", "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "params_b": 405.0, "active_params_b": 405.0, "precision": "FP8", "gpu": "H100", "num_gpus": 8, "wh_per_million_output_tokens": 572.94, "joules_per_output_token": 2.0625855, "median_itl_ms": 66.9, "avg_requests_in_flight": 255.2, "gpu_power_watts_total": 4875, "avg_output_tokens": 367, "avg_running_batch": 255.2, "max_num_seqs": 256, "row_id": "llama-3-1-405b-instruct-fp8/H100/8/256" } ] }