{ "version": "1", "updated": "2026-10-19", "data_version": "2026-10-19", "unit": "Wh per million output tokens", "itl_deadline_ms": 500, "source": { "name": "ML.ENERGY Leaderboard", "publisher": "The ML.ENERGY Initiative (started at SymbioticLab, University of Michigan)", "url": "https://ml.energy/leaderboard/", "data_url": "https://raw.githubusercontent.com/ml-energy/leaderboard/master/public/data/tasks/lm-arena-chat.json", "version": "3.0 (released 2025-12-01); data compiled 2026-02-16", "released": "2025-12-01", "compiled": "2026-02-16", "compiled_evidence": "the leaderboard's public/data/index.json last_updated, and the last commit to the task files, both 2026-02-16", "retrieved": "2026-09-26", "licence": "Apache-2.0 per the ml-energy/benchmark-v3 dataset card", "licence_evidence": [ "Hugging Face dataset ml-energy/benchmark-v3: tagged license:apache-2.0, ships a LICENSE file, and its dataset card's License section reads Apache-2.0", "The ML.ENERGY Leaderboard web app's footer says Apache 2.0", "The leaderboard repository (github.com/ml-energy/leaderboard) carries no licence file; its JSON files are reproduced here as published measurements until the ML.ENERGY team confirms that the dataset licence covers them" ], "workload": "Text Conversation (lm-arena-chat)", "method": "Energy per output token is steady-state GPU energy divided by the output tokens generated in the steady-state window (the benchmark's steady_state_energy_per_token), measured with Zeus under vLLM 0.11.1 on H100 80GB HBM3 and B200 servers. Energy per request is that figure multiplied by the average output length, not a separate measurement. Prompt processing is included.", "selection": "Each index row is the lowest-energy configuration for its model and GPU whose median inter-token latency does not exceed 500 ms, the leaderboard's own default ceiling; no measured configuration reaches it.", "cite": "Chung et al., The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization, NeurIPS 2025 Datasets and Benchmarks, arXiv 2505.06371", "cite_url": "https://arxiv.org/abs/2505.06371", "used_for": "index, per-model configurations, concurrency curves, measured GPU power", "configurations": "328 stable configurations: the leaderboard publishes the runs that passed its stability filter; every run, including the unstable ones, is in the gated Hugging Face dataset ml-energy/benchmark-v3" }, "caveat": "GPU energy only, measured at steady state on a busy server (the most energy-efficient configuration under a latency limit). Host, network, cooling and idle time are not counted. Every figure is a floor; a lightly used server sits well above it.", "boundary": "gpu_only", "slug": "qwen-3-235b-a22b-instruct", "name": "Qwen 3 235B A22B Instruct", "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "params_b": 235.0, "active_params_b": 22.0, "architecture": "MoE", "gpus": [ "B200", "H100" ], "configurations": [ { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 8, "avg_requests_in_flight": 8.0, "wh_per_million_output_tokens": 1793.55, "joules_per_output_token": 6.4567701, "median_itl_ms": 19.7, "p90_itl_ms": 20.3, "p99_itl_ms": 21.4, "tokens_per_second": 396.8, "gpu_power_watts_total": 2562, "avg_output_tokens": 865, "under_itl_deadline": true, "avg_running_batch": 8.0, "p95_itl_ms": 20.5, "row_id": "qwen-3-235b-a22b-instruct/B200/4/8", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 16, "avg_requests_in_flight": 16.0, "wh_per_million_output_tokens": 1264.0, "joules_per_output_token": 4.5503881, "median_itl_ms": 24.5, "p90_itl_ms": 25.7, "p99_itl_ms": 56.3, "tokens_per_second": 636.6, "gpu_power_watts_total": 2897, "avg_output_tokens": 869, "under_itl_deadline": true, "avg_running_batch": 16.0, "p95_itl_ms": 26.1, "row_id": "qwen-3-235b-a22b-instruct/B200/4/16", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 32, "avg_requests_in_flight": 32.0, "wh_per_million_output_tokens": 847.99, "joules_per_output_token": 3.0527544, "median_itl_ms": 29.8, "p90_itl_ms": 30.7, "p99_itl_ms": 54.0, "tokens_per_second": 1043.6, "gpu_power_watts_total": 3186, "avg_output_tokens": 871, "under_itl_deadline": true, "avg_running_batch": 32.0, "p95_itl_ms": 31.0, "row_id": "qwen-3-235b-a22b-instruct/B200/4/32", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 64, "avg_requests_in_flight": 63.9, "wh_per_million_output_tokens": 567.51, "joules_per_output_token": 2.0430333, "median_itl_ms": 40.8, "p90_itl_ms": 42.1, "p99_itl_ms": 67.2, "tokens_per_second": 1528.4, "gpu_power_watts_total": 3123, "avg_output_tokens": 876, "under_itl_deadline": true, "avg_running_batch": 63.9, "p95_itl_ms": 43.6, "row_id": "qwen-3-235b-a22b-instruct/B200/4/64", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 128, "avg_requests_in_flight": 127.9, "wh_per_million_output_tokens": 335.36, "joules_per_output_token": 1.2073054, "median_itl_ms": 47.1, "p90_itl_ms": 50.4, "p99_itl_ms": 85.3, "tokens_per_second": 2603.3, "gpu_power_watts_total": 3143, "avg_output_tokens": 868, "under_itl_deadline": true, "avg_running_batch": 127.9, "p95_itl_ms": 53.1, "row_id": "qwen-3-235b-a22b-instruct/B200/4/128", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 256, "avg_requests_in_flight": 255.7, "wh_per_million_output_tokens": 197.49, "joules_per_output_token": 0.7109701, "median_itl_ms": 53.3, "p90_itl_ms": 58.4, "p99_itl_ms": 105.6, "tokens_per_second": 4506.3, "gpu_power_watts_total": 3204, "avg_output_tokens": 879, "under_itl_deadline": true, "avg_running_batch": 255.7, "p95_itl_ms": 64.3, "row_id": "qwen-3-235b-a22b-instruct/B200/4/256", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 512, "avg_requests_in_flight": 511.4, "wh_per_million_output_tokens": 128.61, "joules_per_output_token": 0.4629842, "median_itl_ms": 67.1, "p90_itl_ms": 106.4, "p99_itl_ms": 130.3, "tokens_per_second": 6506.2, "gpu_power_watts_total": 3012, "avg_output_tokens": 880, "under_itl_deadline": true, "avg_running_batch": 511.4, "p95_itl_ms": 114.1, "row_id": "qwen-3-235b-a22b-instruct/B200/4/512", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 1024, "avg_requests_in_flight": 1022.6, "wh_per_million_output_tokens": 95.47, "joules_per_output_token": 0.3436851, "median_itl_ms": 103.9, "p90_itl_ms": 128.0, "p99_itl_ms": 182.0, "tokens_per_second": 8735.1, "gpu_power_watts_total": 3002, "avg_output_tokens": 868, "under_itl_deadline": true, "avg_running_batch": 1022.6, "p95_itl_ms": 135.2, "row_id": "qwen-3-235b-a22b-instruct/B200/4/1024", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 1536, "avg_requests_in_flight": 1411.2, "wh_per_million_output_tokens": 87.83, "joules_per_output_token": 0.316206, "median_itl_ms": 135.5, "p90_itl_ms": 169.6, "p99_itl_ms": 286.6, "tokens_per_second": 9509.0, "gpu_power_watts_total": 3007, "avg_output_tokens": 880, "under_itl_deadline": true, "avg_running_batch": 1411.2, "p95_itl_ms": 188.2, "row_id": "qwen-3-235b-a22b-instruct/B200/4/1536", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 2048, "avg_requests_in_flight": 2044.8, "wh_per_million_output_tokens": 74.35, "joules_per_output_token": 0.2676524, "median_itl_ms": 143.6, "p90_itl_ms": 177.4, "p99_itl_ms": 353.7, "tokens_per_second": 12211.7, "gpu_power_watts_total": 3268, "avg_output_tokens": 867, "under_itl_deadline": true, "avg_running_batch": 2044.8, "p95_itl_ms": 195.0, "row_id": "qwen-3-235b-a22b-instruct/B200/4/2048", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 3072, "avg_requests_in_flight": 3046.5, "wh_per_million_output_tokens": 68.22, "joules_per_output_token": 0.2455755, "median_itl_ms": 118.5, "p90_itl_ms": 223.0, "p99_itl_ms": 361.0, "tokens_per_second": 13158.6, "gpu_power_watts_total": 3231, "avg_output_tokens": 869, "under_itl_deadline": true, "avg_running_batch": 3046.5, "p95_itl_ms": 238.6, "row_id": "qwen-3-235b-a22b-instruct/B200/4/3072", "repeat": 1 }, { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 4096, "avg_requests_in_flight": 3764.6, "wh_per_million_output_tokens": 66.73, "joules_per_output_token": 0.2402453, "median_itl_ms": 67.8, "p90_itl_ms": 215.1, "p99_itl_ms": 393.2, "tokens_per_second": 13303.3, "gpu_power_watts_total": 3196, "avg_output_tokens": 874, "under_itl_deadline": true, "avg_running_batch": 3764.6, "p95_itl_ms": 282.5, "row_id": "qwen-3-235b-a22b-instruct/B200/4/4096", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 8, "avg_requests_in_flight": 8.0, "wh_per_million_output_tokens": 2332.23, "joules_per_output_token": 8.3960277, "median_itl_ms": 16.6, "p90_itl_ms": 17.0, "p99_itl_ms": 17.9, "tokens_per_second": 469.5, "gpu_power_watts_total": 3942, "avg_output_tokens": 883, "under_itl_deadline": true, "avg_running_batch": 8.0, "p95_itl_ms": 17.2, "row_id": "qwen-3-235b-a22b-instruct/B200/8/8", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 16, "avg_requests_in_flight": 16.0, "wh_per_million_output_tokens": 1550.3, "joules_per_output_token": 5.5810838, "median_itl_ms": 18.7, "p90_itl_ms": 19.1, "p99_itl_ms": 58.1, "tokens_per_second": 824.9, "gpu_power_watts_total": 4604, "avg_output_tokens": 888, "under_itl_deadline": true, "avg_running_batch": 16.0, "p95_itl_ms": 19.3, "row_id": "qwen-3-235b-a22b-instruct/B200/8/16", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 32, "avg_requests_in_flight": 32.0, "wh_per_million_output_tokens": 1166.84, "joules_per_output_token": 4.20063, "median_itl_ms": 28.8, "p90_itl_ms": 29.7, "p99_itl_ms": 62.4, "tokens_per_second": 1078.4, "gpu_power_watts_total": 4530, "avg_output_tokens": 868, "under_itl_deadline": true, "avg_running_batch": 32.0, "p95_itl_ms": 30.0, "row_id": "qwen-3-235b-a22b-instruct/B200/8/32", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 64, "avg_requests_in_flight": 63.9, "wh_per_million_output_tokens": 706.02, "joules_per_output_token": 2.5416779, "median_itl_ms": 33.0, "p90_itl_ms": 33.9, "p99_itl_ms": 57.2, "tokens_per_second": 1876.8, "gpu_power_watts_total": 4770, "avg_output_tokens": 869, "under_itl_deadline": true, "avg_running_batch": 63.9, "p95_itl_ms": 35.5, "row_id": "qwen-3-235b-a22b-instruct/B200/8/64", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 128, "avg_requests_in_flight": 127.8, "wh_per_million_output_tokens": 420.08, "joules_per_output_token": 1.5122788, "median_itl_ms": 36.9, "p90_itl_ms": 39.4, "p99_itl_ms": 82.6, "tokens_per_second": 3305.9, "gpu_power_watts_total": 5000, "avg_output_tokens": 880, "under_itl_deadline": true, "avg_running_batch": 127.8, "p95_itl_ms": 41.4, "row_id": "qwen-3-235b-a22b-instruct/B200/8/128", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 256, "avg_requests_in_flight": 255.7, "wh_per_million_output_tokens": 261.03, "joules_per_output_token": 0.9397195, "median_itl_ms": 43.5, "p90_itl_ms": 48.6, "p99_itl_ms": 97.0, "tokens_per_second": 5403.3, "gpu_power_watts_total": 5078, "avg_output_tokens": 873, "under_itl_deadline": true, "avg_running_batch": 255.7, "p95_itl_ms": 60.0, "row_id": "qwen-3-235b-a22b-instruct/B200/8/256", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 512, "avg_requests_in_flight": 511.4, "wh_per_million_output_tokens": 187.24, "joules_per_output_token": 0.6740768, "median_itl_ms": 58.9, "p90_itl_ms": 98.1, "p99_itl_ms": 120.9, "tokens_per_second": 7220.3, "gpu_power_watts_total": 4867, "avg_output_tokens": 864, "under_itl_deadline": true, "avg_running_batch": 511.4, "p95_itl_ms": 106.3, "row_id": "qwen-3-235b-a22b-instruct/B200/8/512", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 1024, "avg_requests_in_flight": 1022.6, "wh_per_million_output_tokens": 150.61, "joules_per_output_token": 0.5421826, "median_itl_ms": 112.5, "p90_itl_ms": 135.5, "p99_itl_ms": 170.2, "tokens_per_second": 8316.0, "gpu_power_watts_total": 4509, "avg_output_tokens": 861, "under_itl_deadline": true, "avg_running_batch": 1022.6, "p95_itl_ms": 139.6, "row_id": "qwen-3-235b-a22b-instruct/B200/8/1024", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 1536, "avg_requests_in_flight": 1534.0, "wh_per_million_output_tokens": 130.06, "joules_per_output_token": 0.4682317, "median_itl_ms": 143.6, "p90_itl_ms": 171.4, "p99_itl_ms": 232.0, "tokens_per_second": 10208.3, "gpu_power_watts_total": 4780, "avg_output_tokens": 875, "under_itl_deadline": true, "avg_running_batch": 1534.0, "p95_itl_ms": 178.1, "row_id": "qwen-3-235b-a22b-instruct/B200/8/1536", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 2048, "avg_requests_in_flight": 2045.0, "wh_per_million_output_tokens": 116.64, "joules_per_output_token": 0.4198946, "median_itl_ms": 155.6, "p90_itl_ms": 187.3, "p99_itl_ms": 286.1, "tokens_per_second": 11727.2, "gpu_power_watts_total": 4924, "avg_output_tokens": 874, "under_itl_deadline": true, "avg_running_batch": 2045.0, "p95_itl_ms": 200.8, "row_id": "qwen-3-235b-a22b-instruct/B200/8/2048", "repeat": 1 }, { "gpu": "B200", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 4096, "avg_requests_in_flight": 4090.4, "wh_per_million_output_tokens": 103.67, "joules_per_output_token": 0.3732145, "median_itl_ms": 240.9, "p90_itl_ms": 320.6, "p99_itl_ms": 789.1, "tokens_per_second": 14457.1, "gpu_power_watts_total": 5396, "avg_output_tokens": 874, "under_itl_deadline": true, "avg_running_batch": 4090.4, "p95_itl_ms": 367.3, "row_id": "qwen-3-235b-a22b-instruct/B200/8/4096", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 8, "avg_requests_in_flight": 8.0, "wh_per_million_output_tokens": 2025.72, "joules_per_output_token": 7.2925891, "median_itl_ms": 20.2, "p90_itl_ms": 20.9, "p99_itl_ms": 21.8, "tokens_per_second": 387.9, "gpu_power_watts_total": 2829, "avg_output_tokens": 864, "under_itl_deadline": true, "avg_running_batch": 8.0, "p95_itl_ms": 21.1, "row_id": "qwen-3-235b-a22b-instruct/H100/8/8", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 16, "avg_requests_in_flight": 16.0, "wh_per_million_output_tokens": 1438.42, "joules_per_output_token": 5.1783083, "median_itl_ms": 25.3, "p90_itl_ms": 26.4, "p99_itl_ms": 55.9, "tokens_per_second": 615.8, "gpu_power_watts_total": 3189, "avg_output_tokens": 865, "under_itl_deadline": true, "avg_running_batch": 16.0, "p95_itl_ms": 26.6, "row_id": "qwen-3-235b-a22b-instruct/H100/8/16", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 32, "avg_requests_in_flight": 32.0, "wh_per_million_output_tokens": 1062.63, "joules_per_output_token": 3.8254798, "median_itl_ms": 34.7, "p90_itl_ms": 35.7, "p99_itl_ms": 52.5, "tokens_per_second": 907.6, "gpu_power_watts_total": 3472, "avg_output_tokens": 865, "under_itl_deadline": true, "avg_running_batch": 32.0, "p95_itl_ms": 36.0, "row_id": "qwen-3-235b-a22b-instruct/H100/8/32", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 64, "avg_requests_in_flight": 63.9, "wh_per_million_output_tokens": 664.99, "joules_per_output_token": 2.393957, "median_itl_ms": 40.3, "p90_itl_ms": 41.0, "p99_itl_ms": 72.2, "tokens_per_second": 1547.5, "gpu_power_watts_total": 3705, "avg_output_tokens": 879, "under_itl_deadline": true, "avg_running_batch": 63.9, "p95_itl_ms": 41.6, "row_id": "qwen-3-235b-a22b-instruct/H100/8/64", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 96, "avg_requests_in_flight": 95.9, "wh_per_million_output_tokens": 505.31, "joules_per_output_token": 1.8191199, "median_itl_ms": 43.7, "p90_itl_ms": 44.7, "p99_itl_ms": 84.6, "tokens_per_second": 2113.9, "gpu_power_watts_total": 3846, "avg_output_tokens": 872, "under_itl_deadline": true, "avg_running_batch": 95.9, "p95_itl_ms": 47.0, "row_id": "qwen-3-235b-a22b-instruct/H100/8/96", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 128, "avg_requests_in_flight": 127.8, "wh_per_million_output_tokens": 412.84, "joules_per_output_token": 1.4862393, "median_itl_ms": 46.6, "p90_itl_ms": 48.4, "p99_itl_ms": 89.3, "tokens_per_second": 2626.4, "gpu_power_watts_total": 3903, "avg_output_tokens": 863, "under_itl_deadline": true, "avg_running_batch": 127.8, "p95_itl_ms": 53.2, "row_id": "qwen-3-235b-a22b-instruct/H100/8/128", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 192, "avg_requests_in_flight": 191.8, "wh_per_million_output_tokens": 319.04, "joules_per_output_token": 1.1485328, "median_itl_ms": 52.4, "p90_itl_ms": 54.9, "p99_itl_ms": 101.4, "tokens_per_second": 3471.8, "gpu_power_watts_total": 3987, "avg_output_tokens": 880, "under_itl_deadline": true, "avg_running_batch": 191.8, "p95_itl_ms": 67.5, "row_id": "qwen-3-235b-a22b-instruct/H100/8/192", "repeat": 1 }, { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 256, "avg_requests_in_flight": 228.5, "wh_per_million_output_tokens": 303.53, "joules_per_output_token": 1.0927101, "median_itl_ms": 55.5, "p90_itl_ms": 61.8, "p99_itl_ms": 184.1, "tokens_per_second": 3695.8, "gpu_power_watts_total": 4038, "avg_output_tokens": 874, "under_itl_deadline": true, "avg_running_batch": 228.5, "p95_itl_ms": 88.4, "row_id": "qwen-3-235b-a22b-instruct/H100/8/256", "repeat": 1 } ], "best": { "B200": { "gpu": "B200", "num_gpus": 4, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 4, "precision": "BF16", "max_num_seqs": 4096, "avg_requests_in_flight": 3764.6, "wh_per_million_output_tokens": 66.73, "joules_per_output_token": 0.2402453, "median_itl_ms": 67.8, "p90_itl_ms": 215.1, "p99_itl_ms": 393.2, "tokens_per_second": 13303.3, "gpu_power_watts_total": 3196, "avg_output_tokens": 874, "under_itl_deadline": true, "avg_running_batch": 3764.6, "p95_itl_ms": 282.5, "row_id": "qwen-3-235b-a22b-instruct/B200/4/4096", "repeat": 1 }, "H100": { "gpu": "H100", "num_gpus": 8, "tensor_parallel": 1, "data_parallel": 1, "expert_parallel": 8, "precision": "BF16", "max_num_seqs": 256, "avg_requests_in_flight": 228.5, "wh_per_million_output_tokens": 303.53, "joules_per_output_token": 1.0927101, "median_itl_ms": 55.5, "p90_itl_ms": 61.8, "p99_itl_ms": 184.1, "tokens_per_second": 3695.8, "gpu_power_watts_total": 4038, "avg_output_tokens": 874, "under_itl_deadline": true, "avg_running_batch": 228.5, "p95_itl_ms": 88.4, "row_id": "qwen-3-235b-a22b-instruct/H100/8/256", "repeat": 1 } } }