framework,version,device,op_name,kernel_source,model,architecture,mla_dtype,kv_cache_dtype,gemm_type,num_heads,batch_size,isl,tp_size,step,latency
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,2,0.1311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,4,0.1319
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,8,0.1317
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,16,0.1346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,32,0.1341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,64,0.1340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,128,0.1329
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,256,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,512,0.1384
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,1024,0.1409
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,2048,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,4096,0.1477
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,8192,0.1524
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,16384,0.1656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,32768,0.1562
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,65536,0.1633
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1,1,1,131072,0.1758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,2,0.1395
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,4,0.1371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,8,0.1403
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,16,0.1381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,32,0.1393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,64,0.1380
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,128,0.1397
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,256,0.1373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,512,0.1415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,1024,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,2048,0.1465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,4096,0.1541
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,8192,0.1620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,16384,0.1564
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,32768,0.1606
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,65536,0.1745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,2,1,1,131072,0.1962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,2,0.1449
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,4,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,8,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,16,0.1445
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,32,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,64,0.1445
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,128,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,256,0.1439
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,512,0.1451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,1024,0.1522
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,2048,0.1575
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,4096,0.1664
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,8192,0.1563
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,16384,0.1646
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,32768,0.1755
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,65536,0.1998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,4,1,1,131072,0.2448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,2,0.1510
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,4,0.1492
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,8,0.1507
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,16,0.1495
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,32,0.1500
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,64,0.1507
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,128,0.1499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,256,0.1506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,512,0.1524
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,1024,0.1620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,2048,0.1697
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,4096,0.1600
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,8192,0.1654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,16384,0.1783
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,32768,0.2031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,65536,0.2481
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,8,1,1,131072,0.3464
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,2,0.1615
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,4,0.1640
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,8,0.1635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,16,0.1634
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,32,0.1633
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,64,0.1637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,128,0.1635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,256,0.1634
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,512,0.1679
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,1024,0.1757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,2048,0.1671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,4096,0.1746
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,8192,0.1882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,16384,0.2149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,32768,0.2652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,65536,0.3687
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,16,1,1,131072,0.5717
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,2,0.1698
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,4,0.1712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,8,0.1707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,16,0.1698
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,32,0.1702
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,64,0.1715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,128,0.1705
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,256,0.1735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,512,0.1788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,1024,0.1820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,2048,0.1893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,4096,0.2051
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,8192,0.2284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,16384,0.2807
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,32768,0.3845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,65536,0.5886
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,32,1,1,131072,0.9965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,2,0.1914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,4,0.1941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,8,0.1935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,16,0.1924
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,32,0.1922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,64,0.1914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,128,0.1914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,256,0.1927
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,512,0.1968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,1024,0.2037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,2048,0.2174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,4096,0.2435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,8192,0.2943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,16384,0.3967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,32768,0.6001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,65536,1.0093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,64,1,1,131072,1.8282
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,2,0.2300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,4,0.2306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,8,0.2312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,16,0.2297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,32,0.2297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,64,0.2323
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,128,0.2308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,256,0.2342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,512,0.2386
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,1024,0.2533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,2048,0.2843
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,4096,0.3342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,8192,0.4362
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,16384,0.6411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,32768,1.0531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,65536,1.8721
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,128,1,1,131072,3.5146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,2,0.3101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,4,0.3105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,8,0.3094
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,16,0.3103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,32,0.3090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,64,0.3114
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,128,0.3109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,256,0.3174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,512,0.3321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,1024,0.3623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,2048,0.4160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,4096,0.5217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,8192,0.7264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,16384,1.1485
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,32768,1.9684
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,65536,3.6217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,256,1,1,131072,6.9277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,2,0.4916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,4,0.4922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,8,0.4938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,16,0.4914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,32,0.4922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,64,0.4923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,128,0.4937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,256,0.5061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,512,0.5340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,1024,0.5839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,2048,0.6851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,4096,0.8706
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,8192,1.2623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,16384,1.9968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,32768,3.5083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,512,1,1,65536,6.5216
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,2,0.8520
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,4,0.8503
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,8,0.8499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,16,0.8503
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,32,0.8497
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,64,0.8487
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,128,0.8552
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,256,0.8779
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,512,0.9298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,1024,1.0273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,2048,1.2273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,4096,1.6037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,8192,2.3571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,16384,3.8634
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,128,1024,1,1,32768,6.8830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,2,0.1054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,4,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,8,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,16,0.1066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,32,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,64,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,128,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,256,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,512,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,1024,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,2048,0.1110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,4096,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,8192,0.1196
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,16384,0.1249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,32768,0.1335
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,65536,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1,1,1,131072,0.1473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,2,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,4,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,8,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,16,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,32,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,64,0.1115
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,128,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,256,0.1112
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,512,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,1024,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,2048,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,4096,0.1230
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,8192,0.1277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,16384,0.1375
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,32768,0.1404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,65536,0.1552
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,2,1,1,131072,0.1788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,2,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,4,0.1135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,8,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,16,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,32,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,64,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,128,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,256,0.1117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,512,0.1175
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,1024,0.1186
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,2048,0.1222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,4096,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,8192,0.1387
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,16384,0.1379
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,32768,0.1538
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,65536,0.1764
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,4,1,1,131072,0.2311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,2,0.1168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,4,0.1175
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,8,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,16,0.1166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,32,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,64,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,128,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,256,0.1167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,512,0.1186
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,1024,0.1252
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,2048,0.1309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,4096,0.1399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,8192,0.1388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,16384,0.1522
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,32768,0.1769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,65536,0.2294
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,8,1,1,131072,0.3327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,2,0.1291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,4,0.1288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,8,0.1284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,16,0.1281
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,32,0.1289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,64,0.1292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,128,0.1292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,256,0.1291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,512,0.1311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,1024,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,2048,0.1492
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,4096,0.1467
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,8192,0.1603
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,16384,0.1858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,32768,0.2360
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,65536,0.3415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,16,1,1,131072,0.5408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,2,0.1412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,4,0.1395
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,8,0.1413
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,16,0.1408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,32,0.1407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,64,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,128,0.1403
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,256,0.1408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,512,0.1460
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,1024,0.1543
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,2048,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,4096,0.1685
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,8192,0.1956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,16384,0.2476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,32768,0.3518
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,65536,0.5539
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,32,1,1,131072,0.9744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,2,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,4,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,8,0.1438
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,16,0.1439
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,32,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,64,0.1435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,128,0.1433
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,256,0.1460
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,512,0.1551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,1024,0.1635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,2048,0.1795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,4096,0.2056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,8192,0.2582
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,16384,0.3622
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,32768,0.5650
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,65536,0.9847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,64,1,1,131072,1.7939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,2,0.1637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,4,0.1644
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,8,0.1639
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,16,0.1636
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,32,0.1643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,64,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,128,0.1654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,256,0.1685
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,512,0.1767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,1024,0.1921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,2048,0.2201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,4096,0.2703
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,8192,0.3738
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,16384,0.5734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,32768,0.9960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,65536,1.8091
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,128,1,1,131072,3.5131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,2,0.2109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,4,0.2121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,8,0.2112
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,16,0.2105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,32,0.2118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,64,0.2110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,128,0.2126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,256,0.2208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,512,0.2350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,1024,0.2649
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,2048,0.3193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,4096,0.4205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,8192,0.6334
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,16384,1.0476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,32768,1.8967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,65536,3.5661
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,256,1,1,131072,7.0145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,2,0.3050
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,4,0.3048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,8,0.3055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,16,0.3050
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,32,0.3045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,64,0.3054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,128,0.3084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,256,0.3249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,512,0.3545
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,1024,0.4076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,2048,0.5244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,4096,0.7425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,8192,1.1692
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,16384,2.0167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,32768,3.7709
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,512,1,1,65536,7.1538
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,2,0.5022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,4,0.5001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,8,0.5025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,16,0.5026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,32,0.5017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,64,0.5018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,128,0.5123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,256,0.5370
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,512,0.5889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,1024,0.6910
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,2048,0.9021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,4096,1.3051
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,8192,2.1069
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,16384,3.7120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,64,1024,1,1,32768,6.9290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,2,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,4,0.0867
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,8,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,16,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,32,0.0871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,64,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,128,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,256,0.0866
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,512,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,1024,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,2048,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,4096,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,8192,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,16384,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,32768,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,65536,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1,1,1,131072,0.1462
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,2,0.0912
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,4,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,8,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,16,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,32,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,64,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,128,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,256,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,512,0.0951
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,1024,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,2048,0.0965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,4096,0.1029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,8192,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,16384,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,32768,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,65536,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,2,1,1,131072,0.1718
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,2,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,4,0.0951
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,8,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,16,0.0954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,32,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,64,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,128,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,256,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,512,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,1024,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,2048,0.1036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,4096,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,8192,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,16384,0.1243
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,32768,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,65536,0.1751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,4,1,1,131072,0.2429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,2,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,4,0.1009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,8,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,16,0.1008
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,32,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,64,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,128,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,256,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,512,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,1024,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,2048,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,4096,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,8192,0.1256
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,16384,0.1453
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,32768,0.1801
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,65536,0.2490
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,8,1,1,131072,0.3834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,2,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,4,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,8,0.1066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,16,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,32,0.1072
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,64,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,128,0.1073
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,256,0.1072
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,512,0.1097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,1024,0.1149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,2048,0.1234
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,4096,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,8192,0.1516
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,16384,0.1872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,32768,0.2648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,65536,0.4152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,16,1,1,131072,0.7193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,2,0.1173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,4,0.1174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,8,0.1173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,16,0.1167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,32,0.1180
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,64,0.1174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,128,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,256,0.1173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,512,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,1024,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,2048,0.1412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,4096,0.1618
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,8192,0.1962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,16384,0.2704
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,32768,0.4244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,65536,0.7269
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,32,1,1,131072,1.3291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,2,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,4,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,8,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,16,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,32,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,64,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,128,0.1269
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,256,0.1272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,512,0.1320
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,1024,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,2048,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,4096,0.1982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,8192,0.2735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,16384,0.4223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,32768,0.7212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,65536,1.3165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,64,1,1,131072,2.5128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,2,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,4,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,8,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,16,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,32,0.1436
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,64,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,128,0.1440
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,256,0.1442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,512,0.1554
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,1024,0.1740
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,2048,0.2129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,4096,0.2878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,8192,0.4366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,16384,0.7347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,32768,1.3396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,65536,2.5669
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,128,1,1,131072,5.0602
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,2,0.1757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,4,0.1755
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,8,0.1754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,16,0.1754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,32,0.1757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,64,0.1762
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,128,0.1759
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,256,0.1789
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,512,0.2004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,1024,0.2391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,2048,0.3152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,4096,0.4629
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,8192,0.7603
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,16384,1.3648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,32768,2.6502
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,65536,5.2251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,256,1,1,131072,10.4838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,2,0.2370
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,4,0.2368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,8,0.2374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,16,0.2364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,32,0.2366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,64,0.2368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,128,0.2372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,256,0.2428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,512,0.2804
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,1024,0.3510
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,2048,0.4885
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,4096,0.7497
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,8192,1.2871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,16384,2.4326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,32768,4.8465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,512,1,1,65536,9.8060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,2,0.3729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,4,0.3719
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,8,0.3715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,16,0.3713
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,32,0.3706
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,64,0.3761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,128,0.3781
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,256,0.3865
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,512,0.4610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,1024,0.5981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,2048,0.8763
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,4096,1.4019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,8192,2.4644
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,16384,4.8125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,32,1024,1,1,32768,9.8492
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,2,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,4,0.0758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,8,0.0780
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,16,0.0759
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,32,0.0756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,64,0.0752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,128,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,256,0.0778
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,512,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,1024,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,2048,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,4096,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,8192,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,16384,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,32768,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,65536,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1,1,1,131072,0.1273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,2,0.0802
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,4,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,8,0.0793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,16,0.0803
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,32,0.0803
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,64,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,128,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,256,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,512,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,1024,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,2048,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,4096,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,8192,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,16384,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,32768,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,65536,0.1337
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,2,1,1,131072,0.1550
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,2,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,4,0.0806
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,8,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,16,0.0810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,32,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,64,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,128,0.0831
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,256,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,512,0.0840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,1024,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,2048,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,4096,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,8192,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,16384,0.1153
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,32768,0.1296
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,65536,0.1511
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,4,1,1,131072,0.2029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,2,0.0882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,4,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,8,0.0869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,16,0.0855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,32,0.0882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,64,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,128,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,256,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,512,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,1024,0.0906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,2048,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,4096,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,8192,0.1195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,16384,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,32768,0.1551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,65536,0.2043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,8,1,1,131072,0.3151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,2,0.0927
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,4,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,8,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,16,0.0926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,32,0.0923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,64,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,128,0.0928
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,256,0.0945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,512,0.0945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,1024,0.1029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,2048,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,4096,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,8192,0.1347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,16384,0.1619
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,32768,0.2097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,65536,0.3096
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,16,1,1,131072,0.5061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,2,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,4,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,8,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,16,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,32,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,64,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,128,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,256,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,512,0.1049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,1024,0.1136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,2048,0.1257
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,4096,0.1410
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,8192,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,16384,0.2175
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,32768,0.3168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,65536,0.5209
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,32,1,1,131072,0.9219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,2,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,4,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,8,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,16,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,32,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,64,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,128,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,256,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,512,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,1024,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,2048,0.1456
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,4096,0.1719
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,8192,0.2222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,16384,0.3220
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,32768,0.5257
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,65536,0.9273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,64,1,1,131072,1.7212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,2,0.1199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,4,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,8,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,16,0.1158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,32,0.1198
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,64,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,128,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,256,0.1290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,512,0.1337
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,1024,0.1469
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,2048,0.1718
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,4096,0.2241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,8192,0.3236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,16384,0.5253
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,32768,0.9279
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,65536,1.7189
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,128,1,1,131072,3.3597
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,2,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,4,0.1391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,8,0.1399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,16,0.1405
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,32,0.1395
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,64,0.1390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,128,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,256,0.1458
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,512,0.1614
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,1024,0.1869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,2048,0.2379
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,4096,0.3412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,8192,0.5383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,16384,0.9470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,32768,1.7667
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,65536,3.3647
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,256,1,1,131072,6.7131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,2,0.1772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,4,0.1760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,8,0.1768
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,16,0.1768
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,32,0.1768
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,64,0.1774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,128,0.1805
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,256,0.1869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,512,0.2173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,1024,0.2688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,2048,0.3746
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,4096,0.5843
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,8192,0.9909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,16384,1.8063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,32768,3.4696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,512,1,1,65536,6.7463
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,2,0.2524
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,4,0.2517
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,8,0.2533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,16,0.2532
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,32,0.2525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,64,0.2536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,128,0.2619
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,256,0.2760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,512,0.3281
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,1024,0.4249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,2048,0.6242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,4096,1.0167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,8192,1.7849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,16384,3.3535
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,16,1024,1,1,32768,6.4528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,2,0.0689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,4,0.0689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,8,0.0696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,16,0.0696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,32,0.0696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,64,0.0697
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,128,0.0692
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,256,0.0711
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,512,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,1024,0.0742
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,2048,0.0748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,4096,0.0749
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,8192,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,16384,0.0840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,32768,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,65536,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1,1,1,131072,0.1219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,2,0.0731
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,4,0.0738
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,8,0.0735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,16,0.0729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,32,0.0732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,64,0.0729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,128,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,256,0.0730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,512,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,1024,0.0761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,2048,0.0760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,4096,0.0767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,8192,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,16384,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,32768,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,65536,0.1249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,2,1,1,131072,0.1479
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,2,0.0760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,4,0.0760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,8,0.0758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,16,0.0755
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,32,0.0752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,64,0.0756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,128,0.0755
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,256,0.0752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,512,0.0804
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,1024,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,2048,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,4096,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,8192,0.0909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,16384,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,32768,0.1231
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,65536,0.1444
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,4,1,1,131072,0.1961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,2,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,4,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,8,0.0801
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,16,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,32,0.0800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,64,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,128,0.0801
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,256,0.0796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,512,0.0808
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,1024,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,2048,0.0872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,4096,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,8192,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,16384,0.1235
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,32768,0.1461
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,65536,0.1975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,8,1,1,131072,0.3015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,2,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,4,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,8,0.0881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,16,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,32,0.0881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,64,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,128,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,256,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,512,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,1024,0.0950
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,2048,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,4096,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,8192,0.1301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,16384,0.1557
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,32768,0.2066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,65536,0.3068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,16,1,1,131072,0.4997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,2,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,4,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,8,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,16,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,32,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,64,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,128,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,256,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,512,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,1024,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,2048,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,4096,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,8192,0.1596
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,16384,0.2095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,32768,0.3114
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,65536,0.5155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,32,1,1,131072,0.9192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,2,0.0972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,4,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,8,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,16,0.0968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,32,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,64,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,128,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,256,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,512,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,1024,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,2048,0.1361
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,4096,0.1604
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,8192,0.2109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,16384,0.3138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,32768,0.5161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,65536,0.9165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,64,1,1,131072,1.7111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,2,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,4,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,8,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,16,0.1090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,32,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,64,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,128,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,256,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,512,0.1243
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,1024,0.1391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,2048,0.1650
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,4096,0.2158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,8192,0.3150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,16384,0.5182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,32768,0.9127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,65536,1.7006
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,128,1,1,131072,3.3374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,2,0.1250
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,4,0.1253
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,8,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,16,0.1241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,32,0.1264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,64,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,128,0.1280
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,256,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,512,0.1472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,1024,0.1718
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,2048,0.2229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,4096,0.3275
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,8192,0.5258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,16384,0.9305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,32768,1.7590
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,65536,3.3484
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,256,1,1,131072,6.6639
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,2,0.1507
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,4,0.1520
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,8,0.1521
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,16,0.1514
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,32,0.1518
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,64,0.1520
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,128,0.1549
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,256,0.1622
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,512,0.1896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,1024,0.2408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,2048,0.3503
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,4096,0.5526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,8192,0.9637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,16384,1.7785
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,32768,3.4489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,512,1,1,65536,6.7062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,2,0.2093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,4,0.2081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,8,0.2079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,16,0.2081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,32,0.2088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,64,0.2082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,128,0.2153
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,256,0.2269
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,512,0.2793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,1024,0.3741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,2048,0.5746
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,4096,0.9633
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,8192,1.7357
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,16384,3.2917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,8,1024,1,1,32768,6.4021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,2,0.0648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,4,0.0646
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,8,0.0647
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,16,0.0652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,32,0.0651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,64,0.0650
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,128,0.0651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,256,0.0649
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,512,0.0668
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,1024,0.0683
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,2048,0.0690
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,4096,0.0696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,8192,0.0711
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,16384,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,32768,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,65536,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1,1,1,131072,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,2,0.0713
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,4,0.0715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,8,0.0716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,16,0.0715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,32,0.0717
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,64,0.0721
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,128,0.0719
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,256,0.0716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,512,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,1024,0.0744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,2048,0.0748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,4096,0.0749
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,8192,0.0799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,16384,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,32768,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,65536,0.1246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,2,1,1,131072,0.1487
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,2,0.0757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,4,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,8,0.0756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,16,0.0755
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,32,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,64,0.0758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,128,0.0748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,256,0.0753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,512,0.0782
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,1024,0.0784
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,2048,0.0778
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,4096,0.0828
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,8192,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,16384,0.1047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,32768,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,65536,0.1465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,4,1,1,131072,0.1958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,2,0.0773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,4,0.0780
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,8,0.0780
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,16,0.0777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,32,0.0781
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,64,0.0767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,128,0.0769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,256,0.0775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,512,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,1024,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,2048,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,4096,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,8192,0.1045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,16384,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,32768,0.1438
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,65536,0.1959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,8,1,1,131072,0.2955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,2,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,4,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,8,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,16,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,32,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,64,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,128,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,256,0.0855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,512,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,1024,0.0902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,2048,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,4096,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,8192,0.1271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,16384,0.1526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,32768,0.2018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,65536,0.3054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,16,1,1,131072,0.4971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,2,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,4,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,8,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,16,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,32,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,64,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,128,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,256,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,512,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,1024,0.1046
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,2048,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,4096,0.1321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,8192,0.1557
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,16384,0.2098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,32768,0.3107
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,65536,0.5149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,32,1,1,131072,0.9186
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,2,0.0965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,4,0.0972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,8,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,16,0.0973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,32,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,64,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,128,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,256,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,512,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,1024,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,2048,0.1365
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,4096,0.1606
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,8192,0.2112
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,16384,0.3122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,32768,0.5141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,65536,0.9140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,64,1,1,131072,1.7042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,2,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,4,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,8,0.1025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,16,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,32,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,64,0.1026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,128,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,256,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,512,0.1170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,1024,0.1338
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,2048,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,4096,0.2120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,8192,0.3135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,16384,0.5144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,32768,0.9135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,65536,1.6963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,128,1,1,131072,3.3349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,2,0.1152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,4,0.1177
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,8,0.1156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,16,0.1162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,32,0.1154
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,64,0.1156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,128,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,256,0.1264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,512,0.1410
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,1024,0.1659
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,2048,0.2166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,4096,0.3216
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,8192,0.5211
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,16384,0.9220
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,32768,1.7457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,65536,3.3459
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,256,1,1,131072,6.6500
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,2,0.1416
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,4,0.1407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,8,0.1422
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,16,0.1421
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,32,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,64,0.1421
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,128,0.1452
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,256,0.1525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,512,0.1791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,1024,0.2294
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,2048,0.3377
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,4096,0.5440
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,8192,0.9563
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,16384,1.7667
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,32768,3.4284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,512,1,1,65536,6.6976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,2,0.1869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,4,0.1875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,8,0.1876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,16,0.1874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,32,0.1875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,64,0.1863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,128,0.1934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,256,0.2041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,512,0.2556
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,1024,0.3488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,2048,0.5487
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,4096,0.9371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,8192,1.7097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,16384,3.2703
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,4,1024,1,1,32768,6.3714
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,2,0.0648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,4,0.0651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,8,0.0650
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,16,0.0652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,32,0.0655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,64,0.0655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,128,0.0655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,256,0.0653
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,512,0.0672
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,1024,0.0695
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,2048,0.0694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,4096,0.0699
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,8192,0.0707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,16384,0.0758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,32768,0.0863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,65536,0.1049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1,1,1,131072,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,2,0.0685
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,4,0.0677
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,8,0.0669
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,16,0.0684
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,32,0.0676
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,64,0.0688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,128,0.0692
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,256,0.0673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,512,0.0707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,1024,0.0706
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,2048,0.0712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,4096,0.0714
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,8192,0.0758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,16384,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,32768,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,65536,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,2,1,1,131072,0.1457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,2,0.0706
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,4,0.0713
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,8,0.0716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,16,0.0714
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,32,0.0709
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,64,0.0716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,128,0.0717
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,256,0.0716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,512,0.0733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,1024,0.0735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,2048,0.0749
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,4096,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,8192,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,16384,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,32768,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,65536,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,4,1,1,131072,0.1939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,2,0.0767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,4,0.0777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,8,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,16,0.0771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,32,0.0768
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,64,0.0772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,128,0.0766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,256,0.0772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,512,0.0788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,1024,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,2048,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,4096,0.0848
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,8192,0.0988
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,16384,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,32768,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,65536,0.1941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,8,1,1,131072,0.2946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,2,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,4,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,8,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,16,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,32,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,64,0.0850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,128,0.0842
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,256,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,512,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,1024,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,2048,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,4096,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,8192,0.1248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,16384,0.1506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,32768,0.1991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,65536,0.3010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,16,1,1,131072,0.4953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,2,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,4,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,8,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,16,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,32,0.0890
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,64,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,128,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,256,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,512,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,1024,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,2048,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,4096,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,8192,0.1530
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,16384,0.2037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,32768,0.3067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,65536,0.5105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,32,1,1,131072,0.9111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,2,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,4,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,8,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,16,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,32,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,64,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,128,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,256,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,512,0.0986
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,1024,0.1173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,2048,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,4096,0.1585
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,8192,0.2094
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,16384,0.3110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,32768,0.5114
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,65536,0.9159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,64,1,1,131072,1.7114
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,2,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,4,0.1029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,8,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,16,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,32,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,64,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,128,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,256,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,512,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,1024,0.1334
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,2048,0.1610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,4096,0.2105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,8192,0.3105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,16384,0.5122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,32768,0.9120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,65536,1.6950
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,128,1,1,131072,3.3459
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,2,0.1107
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,4,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,8,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,16,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,32,0.1108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,64,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,128,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,256,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,512,0.1382
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,1024,0.1634
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,2048,0.2136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,4096,0.3179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,8192,0.5184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,16384,0.9214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,32768,1.7447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,65536,3.3426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,256,1,1,131072,6.6641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,2,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,4,0.1364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,8,0.1361
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,16,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,32,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,64,0.1358
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,128,0.1389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,256,0.1473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,512,0.1735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,1024,0.2238
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,2048,0.3300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,4096,0.5358
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,8192,0.9466
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,16384,1.7608
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,32768,3.4230
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,512,1,1,65536,6.6783
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,2,0.1771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,4,0.1783
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,8,0.1770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,16,0.1777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,32,0.1774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,64,0.1779
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,128,0.1833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,256,0.1959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,512,0.2457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,1024,0.3375
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,2048,0.5380
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,4096,0.9292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,8192,1.7009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,16384,3.2580
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,2,1024,1,1,32768,6.3566
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,2,0.0594
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,4,0.0597
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,8,0.0593
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,16,0.0592
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,32,0.0625
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,64,0.0637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,128,0.0635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,256,0.0631
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,512,0.0657
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,1024,0.0675
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,2048,0.0674
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,4096,0.0688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,8192,0.0695
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,16384,0.0741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,32768,0.0827
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,65536,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1,1,1,131072,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,2,0.0665
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,4,0.0670
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,8,0.0671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,16,0.0668
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,32,0.0676
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,64,0.0645
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,128,0.0650
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,256,0.0648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,512,0.0689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,1024,0.0690
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,2048,0.0693
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,4096,0.0698
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,8192,0.0728
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,16384,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,32768,0.0964
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,65536,0.1167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,2,1,1,131072,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,2,0.0692
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,4,0.0691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,8,0.0687
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,16,0.0688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,32,0.0689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,64,0.0687
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,128,0.0689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,256,0.0690
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,512,0.0702
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,1024,0.0712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,2048,0.0713
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,4096,0.0753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,8192,0.0781
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,16384,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,32768,0.1169
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,65536,0.1400
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,4,1,1,131072,0.1897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,2,0.0729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,4,0.0739
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,8,0.0734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,16,0.0739
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,32,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,64,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,128,0.0734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,256,0.0735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,512,0.0755
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,1024,0.0775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,2048,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,4096,0.0831
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,8192,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,16384,0.1171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,32768,0.1416
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,65536,0.1899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,8,1,1,131072,0.2906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,2,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,4,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,8,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,16,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,32,0.0839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,64,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,128,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,256,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,512,0.0855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,1024,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,2048,0.0905
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,4096,0.1036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,8192,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,16384,0.1481
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,32768,0.1982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,65536,0.2988
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,16,1,1,131072,0.4947
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,2,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,4,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,8,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,16,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,32,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,64,0.0889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,128,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,256,0.0893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,512,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,1024,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,2048,0.1071
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,4096,0.1273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,8192,0.1502
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,16384,0.2027
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,32768,0.3025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,65536,0.5051
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,32,1,1,131072,0.9139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,2,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,4,0.0904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,8,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,16,0.0908
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,32,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,64,0.0907
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,128,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,256,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,512,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,1024,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,2048,0.1282
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,4096,0.1542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,8192,0.2027
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,16384,0.3039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,32768,0.5066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,65536,0.9111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,64,1,1,131072,1.7030
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,2,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,4,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,8,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,16,0.0970
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,32,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,64,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,128,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,256,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,512,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,1024,0.1289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,2048,0.1562
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,4096,0.2075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,8192,0.3085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,16384,0.5081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,32768,0.9077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,65536,1.6940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,128,1,1,131072,3.3396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,2,0.1072
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,4,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,8,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,16,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,32,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,64,0.1071
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,128,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,256,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,512,0.1335
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,1024,0.1594
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,2048,0.2092
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,4096,0.3151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,8192,0.5131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,16384,0.9164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,32768,1.7421
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,65536,3.3263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,256,1,1,131072,6.6606
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,2,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,4,0.1297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,8,0.1294
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,16,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,32,0.1291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,64,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,128,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,256,0.1409
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,512,0.1694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,1024,0.2180
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,2048,0.3209
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,4096,0.5318
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,8192,0.9339
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,16384,1.7490
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,32768,3.4048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,512,1,1,65536,6.6836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,2,0.1716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,4,0.1702
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,8,0.1712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,16,0.1712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,32,0.1715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,64,0.1714
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,128,0.1772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,256,0.1892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,512,0.2381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,1024,0.3297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,2048,0.5326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,4096,0.9163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,8192,1.6961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,16384,3.2438
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,bfloat16,1,1024,1,1,32768,6.3471
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,2,0.1297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,4,0.1280
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,8,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,16,0.1282
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,32,0.1291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,64,0.1283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,128,0.1281
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,256,0.1279
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,512,0.1289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,1024,0.1315
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,2048,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,4096,0.1352
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,8192,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,16384,0.1452
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,32768,0.1495
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,65536,0.1530
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1,1,1,131072,0.1605
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,2,0.1362
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,4,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,8,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,16,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,32,0.1359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,64,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,128,0.1346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,256,0.1340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,512,0.1374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,1024,0.1375
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,2048,0.1405
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,4096,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,8192,0.1472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,16384,0.1514
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,32768,0.1524
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,65536,0.1603
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,2,1,1,131072,0.1716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,2,0.1373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,4,0.1352
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,8,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,16,0.1339
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,32,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,64,0.1368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,128,0.1359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,256,0.1335
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,512,0.1380
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,1024,0.1392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,2048,0.1438
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,4096,0.1443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,8192,0.1491
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,16384,0.1509
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,32768,0.1564
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,65536,0.1680
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,4,1,1,131072,0.1933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,2,0.1410
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,4,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,8,0.1422
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,16,0.1433
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,32,0.1419
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,64,0.1422
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,128,0.1396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,256,0.1425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,512,0.1420
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,1024,0.1484
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,2048,0.1506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,4096,0.1536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,8192,0.1542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,16384,0.1625
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,32768,0.1725
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,65536,0.1971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,8,1,1,131072,0.2427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,2,0.1522
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,4,0.1517
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,8,0.1516
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,16,0.1508
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,32,0.1539
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,64,0.1504
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,128,0.1529
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,256,0.1511
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,512,0.1558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,1024,0.1578
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,2048,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,4096,0.1619
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,8192,0.1701
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,16384,0.1816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,32768,0.2082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,65536,0.2574
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,16,1,1,131072,0.3607
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,2,0.1626
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,4,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,8,0.1621
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,16,0.1648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,32,0.1615
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,64,0.1655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,128,0.1623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,256,0.1643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,512,0.1687
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,1024,0.1732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,2048,0.1736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,4096,0.1832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,8192,0.1947
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,16384,0.2230
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,32768,0.2708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,65536,0.3762
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,32,1,1,131072,0.5774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,2,0.1763
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,4,0.1742
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,8,0.1765
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,16,0.1750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,32,0.1766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,64,0.1736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,128,0.1763
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,256,0.1750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,512,0.1791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,1024,0.1798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,2048,0.1904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,4096,0.2000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,8192,0.2283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,16384,0.2766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,32768,0.3818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,65536,0.5868
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,64,1,1,131072,0.9947
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,2,0.1903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,4,0.1904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,8,0.1902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,16,0.1891
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,32,0.1902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,64,0.1903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,128,0.1903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,256,0.1910
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,512,0.1949
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,1024,0.2017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,2048,0.2133
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,4096,0.2431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,8192,0.2921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,16384,0.3972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,32768,0.6053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,65536,1.0211
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,128,1,1,131072,1.8308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,2,0.2392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,4,0.2372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,8,0.2407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,16,0.2381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,32,0.2389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,64,0.2382
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,128,0.2401
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,256,0.2406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,512,0.2482
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,1024,0.2599
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,2048,0.2892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,4096,0.3442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,8192,0.4475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,16384,0.6612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,32768,1.0787
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,65536,1.9057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,256,1,1,131072,3.5446
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,2,0.3372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,4,0.3381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,8,0.3382
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,16,0.3380
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,32,0.3389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,64,0.3370
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,128,0.3372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,256,0.3428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,512,0.3561
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,1024,0.3799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,2048,0.4320
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,4096,0.5306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,8192,0.7173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,16384,1.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,32768,1.8272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,512,1,1,65536,3.3099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,2,0.5528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,4,0.5540
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,8,0.5533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,16,0.5543
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,32,0.5532
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,64,0.5534
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,128,0.5529
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,256,0.5591
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,512,0.5854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,1024,0.6371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,2048,0.7366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,4096,0.9321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,8192,1.3022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,16384,2.0392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,128,1024,1,1,32768,3.5189
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,2,0.1012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,4,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,8,0.1010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,16,0.1012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,32,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,64,0.1011
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,128,0.1013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,256,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,512,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,1024,0.1054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,2048,0.1055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,4096,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,8192,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,16384,0.1134
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,32768,0.1166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,65536,0.1217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1,1,1,131072,0.1300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,2,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,4,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,8,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,16,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,32,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,64,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,128,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,256,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,512,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,1024,0.1111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,2048,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,4096,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,8192,0.1151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,16384,0.1219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,32768,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,65536,0.1374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,2,1,1,131072,0.1479
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,2,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,4,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,8,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,16,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,32,0.1108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,64,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,128,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,256,0.1089
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,512,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,1024,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,2048,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,4096,0.1184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,8192,0.1238
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,16384,0.1280
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,32768,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,65536,0.1455
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,4,1,1,131072,0.1691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,2,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,4,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,8,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,16,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,32,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,64,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,128,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,256,0.1136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,512,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,1024,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,2048,0.1194
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,4096,0.1243
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,8192,0.1282
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,16384,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,32768,0.1445
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,65536,0.1674
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,8,1,1,131072,0.2104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,2,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,4,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,8,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,16,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,32,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,64,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,128,0.1218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,256,0.1216
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,512,0.1231
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,1024,0.1277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,2048,0.1329
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,4096,0.1352
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,8192,0.1416
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,16384,0.1517
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,32768,0.1748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,65536,0.2192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,16,1,1,131072,0.3086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,2,0.1299
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,4,0.1300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,8,0.1317
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,16,0.1322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,32,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,64,0.1309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,128,0.1301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,256,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,512,0.1335
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,1024,0.1368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,2048,0.1410
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,4096,0.1473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,8192,0.1594
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,16384,0.1837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,32768,0.2311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,65536,0.3244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,32,1,1,131072,0.5157
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,2,0.1326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,4,0.1325
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,8,0.1330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,16,0.1327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,32,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,64,0.1328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,128,0.1333
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,256,0.1346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,512,0.1422
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,1024,0.1455
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,2048,0.1514
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,4096,0.1643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,8192,0.1876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,16384,0.2361
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,32768,0.3284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,65536,0.5222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,64,1,1,131072,0.8962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,2,0.1457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,4,0.1467
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,8,0.1454
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,16,0.1453
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,32,0.1458
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,64,0.1465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,128,0.1466
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,256,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,512,0.1512
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,1024,0.1576
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,2048,0.1708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,4096,0.1935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,8192,0.2426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,16384,0.3353
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,32768,0.5321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,65536,0.9095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,128,1,1,131072,1.6595
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,2,0.1761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,4,0.1758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,8,0.1771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,16,0.1774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,32,0.1770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,64,0.1771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,128,0.1770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,256,0.1801
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,512,0.1870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,1024,0.2001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,2048,0.2267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,4096,0.2753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,8192,0.3711
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,16384,0.5694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,32768,0.9539
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,65536,1.7108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,256,1,1,131072,3.2238
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,2,0.2447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,4,0.2420
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,8,0.2424
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,16,0.2426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,32,0.2426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,64,0.2427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,128,0.2439
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,256,0.2476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,512,0.2609
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,1024,0.2872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,2048,0.3384
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,4096,0.4391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,8192,0.6381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,16384,1.0321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,32768,1.8032
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,512,1,1,65536,3.3457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,2,0.3694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,4,0.3688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,8,0.3688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,16,0.3690
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,32,0.3703
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,64,0.3691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,128,0.3707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,256,0.3834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,512,0.4069
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,1024,0.4571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,2048,0.5506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,4096,0.7328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,8192,1.0885
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,16384,1.7946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,64,1024,1,1,32768,3.2087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,2,0.0930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,4,0.0944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,8,0.0944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,16,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,32,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,64,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,128,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,256,0.0929
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,512,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,1024,0.0969
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,2048,0.0968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,4096,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,8192,0.0992
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,16384,0.1049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,32768,0.1088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,65536,0.1134
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1,1,1,131072,0.1258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,2,0.0967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,4,0.0990
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,8,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,16,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,32,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,64,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,128,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,256,0.0985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,512,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,1024,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,2048,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,4096,0.1066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,8192,0.1069
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,16384,0.1132
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,32768,0.1170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,65536,0.1264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,2,1,1,131072,0.1408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,2,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,4,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,8,0.1029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,16,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,32,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,64,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,128,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,256,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,512,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,1024,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,2048,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,4096,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,8192,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,16384,0.1184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,32768,0.1272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,65536,0.1415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,4,1,1,131072,0.1729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,2,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,4,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,8,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,16,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,32,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,64,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,128,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,256,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,512,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,1024,0.1108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,2048,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,4096,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,8192,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,16384,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,32768,0.1472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,65536,0.1775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,8,1,1,131072,0.2401
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,2,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,4,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,8,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,16,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,32,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,64,0.1135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,128,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,256,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,512,0.1168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,1024,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,2048,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,4096,0.1265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,8192,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,16384,0.1528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,32768,0.1871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,65536,0.2552
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,16,1,1,131072,0.3916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,2,0.1248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,4,0.1244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,8,0.1246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,16,0.1255
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,32,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,64,0.1244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,128,0.1244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,256,0.1244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,512,0.1265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,1024,0.1306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,2048,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,4096,0.1457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,8192,0.1627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,16384,0.1968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,32768,0.2661
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,65536,0.4010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,32,1,1,131072,0.6734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,2,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,4,0.1336
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,8,0.1342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,16,0.1342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,32,0.1347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,64,0.1339
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,128,0.1341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,256,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,512,0.1371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,1024,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,2048,0.1493
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,4096,0.1678
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,8192,0.2015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,16384,0.2707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,32768,0.4041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,65536,0.6753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,64,1,1,131072,1.2169
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,2,0.1558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,4,0.1558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,8,0.1565
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,16,0.1566
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,32,0.1565
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,64,0.1573
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,128,0.1564
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,256,0.1566
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,512,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,1024,0.1691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,2048,0.1883
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,4096,0.2246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,8192,0.2925
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,16384,0.4292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,32768,0.7048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,65536,1.2556
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,128,1,1,131072,2.3641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,2,0.1997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,4,0.2001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,8,0.1998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,16,0.1999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,32,0.2007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,64,0.2003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,128,0.1981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,256,0.1998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,512,0.2080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,1024,0.2269
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,2048,0.2620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,4096,0.3329
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,8192,0.4681
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,16384,0.7388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,32768,1.2863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,65536,2.4192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,256,1,1,131072,4.7046
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,2,0.2794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,4,0.2790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,8,0.2789
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,16,0.2792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,32,0.2795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,64,0.2787
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,128,0.2770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,256,0.2761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,512,0.2923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,1024,0.3261
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,2048,0.3908
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,4096,0.5142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,8192,0.7537
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,16384,1.2359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,32768,2.2375
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,512,1,1,65536,4.3324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,2,0.4707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,4,0.4702
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,8,0.4723
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,16,0.4715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,32,0.4697
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,64,0.4704
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,128,0.4689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,256,0.4694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,512,0.4964
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,1024,0.5612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,2048,0.6861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,4096,0.9346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,8192,1.4128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,16384,2.3736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,32,1024,1,1,32768,4.3691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,2,0.0839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,4,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,8,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,16,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,32,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,64,0.0839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,128,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,256,0.0839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,512,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,1024,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,2048,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,4096,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,8192,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,16384,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,32768,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,65536,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1,1,1,131072,0.1184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,2,0.0870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,4,0.0870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,8,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,16,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,32,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,64,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,128,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,256,0.0870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,512,0.0882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,1024,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,2048,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,4096,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,8192,0.0944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,16384,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,32768,0.1115
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,65536,0.1226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,2,1,1,131072,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,2,0.0891
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,4,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,8,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,16,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,32,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,64,0.0891
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,128,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,256,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,512,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,1024,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,2048,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,4096,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,8192,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,16384,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,32768,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,65536,0.1292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,4,1,1,131072,0.1521
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,2,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,4,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,8,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,16,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,32,0.0927
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,64,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,128,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,256,0.0926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,512,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,1024,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,2048,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,4096,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,8192,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,16384,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,32768,0.1317
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,65536,0.1525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,8,1,1,131072,0.1951
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,2,0.1010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,4,0.0987
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,8,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,16,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,32,0.0973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,64,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,128,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,256,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,512,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,1024,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,2048,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,4096,0.1156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,8192,0.1247
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,16384,0.1353
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,32768,0.1579
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,65536,0.1984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,16,1,1,131072,0.2817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,2,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,4,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,8,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,16,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,32,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,64,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,128,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,256,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,512,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,1024,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,2048,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,4096,0.1271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,8192,0.1391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,16384,0.1613
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,32768,0.2055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,65536,0.2926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,32,1,1,131072,0.4691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,2,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,4,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,8,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,16,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,32,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,64,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,128,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,256,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,512,0.1190
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,1024,0.1269
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,2048,0.1340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,4096,0.1452
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,8192,0.1677
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,16384,0.2123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,32768,0.3007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,65536,0.4761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,64,1,1,131072,0.8229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,2,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,4,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,8,0.1265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,16,0.1264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,32,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,64,0.1268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,128,0.1279
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,256,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,512,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,1024,0.1416
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,2048,0.1531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,4096,0.1756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,8192,0.2192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,16384,0.3066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,32768,0.4811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,65536,0.8281
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,128,1,1,131072,1.5242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,2,0.1536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,4,0.1531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,8,0.1528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,16,0.1529
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,32,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,64,0.1527
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,128,0.1541
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,256,0.1553
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,512,0.1607
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,1024,0.1728
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,2048,0.1976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,4096,0.2437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,8192,0.3328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,16384,0.5069
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,32768,0.8568
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,65536,1.5624
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,256,1,1,131072,2.9646
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,2,0.2011
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,4,0.2002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,8,0.2017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,16,0.2003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,32,0.2004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,64,0.2002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,128,0.2003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,256,0.2042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,512,0.2151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,1024,0.2408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,2048,0.2864
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,4096,0.3774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,8192,0.5569
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,16384,0.9145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,32768,1.6297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,512,1,1,65536,3.0620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,2,0.2916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,4,0.2922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,8,0.2920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,16,0.2920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,32,0.2912
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,64,0.2907
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,128,0.2921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,256,0.3012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,512,0.3263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,1024,0.3689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,2048,0.4547
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,4096,0.6225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,8192,0.9572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,16384,1.6204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,16,1024,1,1,32768,2.9492
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,2,0.0773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,4,0.0770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,8,0.0770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,16,0.0775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,32,0.0767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,64,0.0770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,128,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,256,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,512,0.0775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,1024,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,2048,0.0797
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,4096,0.0806
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,8192,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,16384,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,32768,0.0904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,65536,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1,1,1,131072,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,2,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,4,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,8,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,16,0.0807
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,32,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,64,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,128,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,256,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,512,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,1024,0.0831
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,2048,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,4096,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,8192,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,16384,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,32768,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,65536,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,2,1,1,131072,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,2,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,4,0.0829
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,8,0.0819
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,16,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,32,0.0830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,64,0.0830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,128,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,256,0.0825
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,512,0.0841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,1024,0.0849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,2048,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,4096,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,8192,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,16384,0.0948
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,32768,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,65536,0.1209
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,4,1,1,131072,0.1437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,2,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,4,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,8,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,16,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,32,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,64,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,128,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,256,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,512,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,1024,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,2048,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,4096,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,8192,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,16384,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,32768,0.1240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,65536,0.1459
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,8,1,1,131072,0.1873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,2,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,4,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,8,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,16,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,32,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,64,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,128,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,256,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,512,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,1024,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,2048,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,4096,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,8192,0.1192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,16384,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,32768,0.1548
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,65536,0.1956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,16,1,1,131072,0.2782
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,2,0.1026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,4,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,8,0.1027
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,16,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,32,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,64,0.1032
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,128,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,256,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,512,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,1024,0.1052
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,2048,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,4096,0.1230
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,8192,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,16384,0.1573
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,32768,0.2016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,65536,0.2876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,32,1,1,131072,0.4632
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,2,0.1055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,4,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,8,0.1052
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,16,0.1054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,32,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,64,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,128,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,256,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,512,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,1024,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,2048,0.1253
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,4096,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,8192,0.1589
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,16384,0.2041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,32768,0.2902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,65536,0.4661
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,64,1,1,131072,0.8128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,2,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,4,0.1130
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,8,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,16,0.1135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,32,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,64,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,128,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,256,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,512,0.1199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,1024,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,2048,0.1420
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,4096,0.1644
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,8192,0.2080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,16384,0.2957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,32768,0.4712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,65536,0.8152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,128,1,1,131072,1.5106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,2,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,4,0.1296
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,8,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,16,0.1300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,32,0.1300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,64,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,128,0.1316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,256,0.1349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,512,0.1410
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,1024,0.1528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,2048,0.1759
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,4096,0.2219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,8192,0.3099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,16384,0.4842
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,32768,0.8400
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,65536,1.5433
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,256,1,1,131072,2.9442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,2,0.1616
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,4,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,8,0.1609
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,16,0.1623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,32,0.1614
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,64,0.1610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,128,0.1625
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,256,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,512,0.1756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,1024,0.2019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,2048,0.2471
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,4096,0.3383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,8192,0.5156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,16384,0.8740
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,32768,1.5886
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,512,1,1,65536,3.0203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,2,0.2227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,4,0.2224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,8,0.2227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,16,0.2225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,32,0.2224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,64,0.2236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,128,0.2228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,256,0.2289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,512,0.2542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,1024,0.2970
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,2048,0.3824
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,4096,0.5513
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,8192,0.8836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,16384,1.5471
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,8,1024,1,1,32768,2.8763
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,2,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,4,0.0708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,8,0.0707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,16,0.0711
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,32,0.0709
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,64,0.0698
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,128,0.0715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,256,0.0709
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,512,0.0718
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,1024,0.0730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,2048,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,4096,0.0743
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,8192,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,16384,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,32768,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,65536,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1,1,1,131072,0.1046
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,2,0.0796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,4,0.0793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,8,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,16,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,32,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,64,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,128,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,256,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,512,0.0797
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,1024,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,2048,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,4096,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,8192,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,16384,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,32768,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,65536,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,2,1,1,131072,0.1252
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,2,0.0831
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,4,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,8,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,16,0.0830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,32,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,64,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,128,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,256,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,512,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,1024,0.0839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,2048,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,4096,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,8192,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,16384,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,32768,0.1091
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,65536,0.1207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,4,1,1,131072,0.1435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,2,0.0846
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,4,0.0849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,8,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,16,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,32,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,64,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,128,0.0849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,256,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,512,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,1024,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,2048,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,4096,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,8192,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,16384,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,32768,0.1195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,65536,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,8,1,1,131072,0.1848
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,2,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,4,0.0931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,8,0.0945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,16,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,32,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,64,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,128,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,256,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,512,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,1024,0.0965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,2048,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,4096,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,8192,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,16384,0.1274
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,32768,0.1501
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,65536,0.1903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,16,1,1,131072,0.2735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,2,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,4,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,8,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,16,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,32,0.1010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,64,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,128,0.1011
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,256,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,512,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,1024,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,2048,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,4096,0.1177
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,8192,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,16384,0.1541
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,32768,0.1975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,65536,0.2848
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,32,1,1,131072,0.4613
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,2,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,4,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,8,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,16,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,32,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,64,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,128,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,256,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,512,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,1024,0.1092
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,2048,0.1191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,4096,0.1322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,8192,0.1550
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,16384,0.1985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,32768,0.2863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,65536,0.4620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,64,1,1,131072,0.8131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,2,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,4,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,8,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,16,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,32,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,64,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,128,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,256,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,512,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,1024,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,2048,0.1359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,4096,0.1582
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,8192,0.2020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,16384,0.2881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,32768,0.4642
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,65536,0.8106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,128,1,1,131072,1.5033
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,2,0.1213
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,4,0.1197
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,8,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,16,0.1192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,32,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,64,0.1194
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,128,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,256,0.1217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,512,0.1321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,1024,0.1437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,2048,0.1667
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,4096,0.2123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,8192,0.2997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,16384,0.4748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,32768,0.8240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,65536,1.5300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,256,1,1,131072,2.9264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,2,0.1426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,4,0.1409
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,8,0.1417
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,16,0.1407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,32,0.1423
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,64,0.1414
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,128,0.1437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,256,0.1459
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,512,0.1585
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,1024,0.1820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,2048,0.2280
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,4096,0.3180
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,8192,0.4959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,16384,0.8540
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,32768,1.5696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,512,1,1,65536,2.9998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,2,0.1897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,4,0.1893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,8,0.1889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,16,0.1894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,32,0.1897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,64,0.1886
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,128,0.1896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,256,0.1950
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,512,0.2185
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,1024,0.2612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,2048,0.3462
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,4096,0.5138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,8192,0.8465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,16384,1.5091
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,4,1024,1,1,32768,2.8379
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,2,0.0691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,4,0.0677
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,8,0.0673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,16,0.0677
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,32,0.0713
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,64,0.0709
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,128,0.0727
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,256,0.0724
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,512,0.0732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,1024,0.0752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,2048,0.0753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,4096,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,8192,0.0770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,16384,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,32768,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,65536,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1,1,1,131072,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,2,0.0748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,4,0.0741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,8,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,16,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,32,0.0738
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,64,0.0734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,128,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,256,0.0739
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,512,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,1024,0.0758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,2048,0.0758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,4096,0.0769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,8192,0.0800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,16384,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,32768,0.0871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,65536,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,2,1,1,131072,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,2,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,4,0.0789
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,8,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,16,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,32,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,64,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,128,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,256,0.0807
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,512,0.0799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,1024,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,2048,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,4096,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,8192,0.0849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,16384,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,32768,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,65536,0.1176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,4,1,1,131072,0.1395
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,2,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,4,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,8,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,16,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,32,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,64,0.0863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,128,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,256,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,512,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,1024,0.0881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,2048,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,4096,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,8192,0.0932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,16384,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,32768,0.1207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,65536,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,8,1,1,131072,0.1859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,2,0.0910
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,4,0.0910
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,8,0.0932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,16,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,32,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,64,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,128,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,256,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,512,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,1024,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,2048,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,4096,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,8192,0.1095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,16384,0.1240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,32768,0.1475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,65536,0.1880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,16,1,1,131072,0.2710
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,2,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,4,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,8,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,16,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,32,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,64,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,128,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,256,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,512,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,1024,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,2048,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,4096,0.1145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,8192,0.1282
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,16384,0.1517
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,32768,0.1945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,65536,0.2825
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,32,1,1,131072,0.4575
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,2,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,4,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,8,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,16,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,32,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,64,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,128,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,256,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,512,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,1024,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,2048,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,4096,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,8192,0.1519
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,16384,0.1963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,32768,0.2847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,65536,0.4606
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,64,1,1,131072,0.8093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,2,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,4,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,8,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,16,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,32,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,64,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,128,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,256,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,512,0.1097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,1024,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,2048,0.1329
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,4096,0.1553
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,8192,0.1993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,16384,0.2870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,32768,0.4634
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,65536,0.8092
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,128,1,1,131072,1.5039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,2,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,4,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,8,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,16,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,32,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,64,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,128,0.1154
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,256,0.1178
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,512,0.1277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,1024,0.1386
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,2048,0.1616
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,4096,0.2074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,8192,0.2947
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,16384,0.4700
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,32768,0.8197
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,65536,1.5261
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,256,1,1,131072,2.9272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,2,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,4,0.1327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,8,0.1325
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,16,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,32,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,64,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,128,0.1349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,256,0.1384
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,512,0.1494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,1024,0.1725
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,2048,0.2182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,4096,0.3087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,8192,0.4868
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,16384,0.8428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,32768,1.5581
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,512,1,1,65536,2.9896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,2,0.1727
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,4,0.1723
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,8,0.1752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,16,0.1720
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,32,0.1727
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,64,0.1720
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,128,0.1739
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,256,0.1779
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,512,0.2010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,1024,0.2429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,2048,0.3280
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,4096,0.4946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,8192,0.8272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,16384,1.4901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,2,1024,1,1,32768,2.8193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,2,0.0658
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,4,0.0649
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,8,0.0653
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,16,0.0654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,32,0.0653
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,64,0.0655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,128,0.0662
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,256,0.0657
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,512,0.0671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,1024,0.0733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,2048,0.0737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,4096,0.0733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,8192,0.0747
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,16384,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,32768,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,65536,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1,1,1,131072,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,2,0.0730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,4,0.0731
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,8,0.0731
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,16,0.0731
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,32,0.0743
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,64,0.0729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,128,0.0732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,256,0.0730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,512,0.0733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,1024,0.0740
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,2048,0.0752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,4096,0.0748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,8192,0.0782
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,16384,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,32768,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,65536,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,2,1,1,131072,0.1171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,2,0.0752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,4,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,8,0.0749
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,16,0.0750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,32,0.0747
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,64,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,128,0.0749
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,256,0.0747
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,512,0.0755
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,1024,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,2048,0.0770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,4096,0.0796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,8192,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,16384,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,32768,0.0973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,65536,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,4,1,1,131072,0.1360
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,2,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,4,0.0810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,8,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,16,0.0806
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,32,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,64,0.0799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,128,0.0803
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,256,0.0808
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,512,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,1024,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,2048,0.0849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,4096,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,8192,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,16384,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,32768,0.1168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,65536,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,8,1,1,131072,0.1795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,2,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,4,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,8,0.0902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,16,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,32,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,64,0.0902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,128,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,256,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,512,0.0925
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,1024,0.0931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,2048,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,4096,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,8192,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,16384,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,32768,0.1449
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,65536,0.1859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,16,1,1,131072,0.2688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,2,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,4,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,8,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,16,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,32,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,64,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,128,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,256,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,512,0.0972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,1024,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,2048,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,4096,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,8192,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,16384,0.1491
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,32768,0.1921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,65536,0.2792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,32,1,1,131072,0.4545
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,2,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,4,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,8,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,16,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,32,0.0970
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,64,0.0968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,128,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,256,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,512,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,1024,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,2048,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,4096,0.1265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,8192,0.1488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,16384,0.1933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,32768,0.2804
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,65536,0.4573
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,64,1,1,131072,0.8056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,2,0.1032
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,4,0.1025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,8,0.1034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,16,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,32,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,64,0.1034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,128,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,256,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,512,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,1024,0.1158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,2048,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,4096,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,8192,0.1958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,16384,0.2833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,32768,0.4588
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,65536,0.8060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,128,1,1,131072,1.4988
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,2,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,4,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,8,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,16,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,32,0.1115
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,64,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,128,0.1132
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,256,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,512,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,1024,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,2048,0.1574
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,4096,0.2036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,8192,0.2919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,16384,0.4661
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,32768,0.8164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,65536,1.5210
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,256,1,1,131072,2.9213
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,2,0.1245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,4,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,8,0.1240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,16,0.1258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,32,0.1247
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,64,0.1259
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,128,0.1283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,256,0.1312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,512,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,1024,0.1660
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,2048,0.2101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,4096,0.3017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,8192,0.4800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,16384,0.8369
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,32768,1.5525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,512,1,1,65536,2.9816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,2,0.1611
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,4,0.1623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,8,0.1624
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,16,0.1620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,32,0.1627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,64,0.1621
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,128,0.1643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,256,0.1680
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,512,0.1903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,1024,0.2321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,2048,0.3165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,4096,0.4826
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,8192,0.8161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,16384,1.4802
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,bfloat16,1,1024,1,1,32768,2.8075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,2,0.1359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,4,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,8,0.1365
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,16,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,32,0.1370
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,64,0.1360
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,128,0.1361
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,256,0.1355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,512,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,1024,0.1415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,2048,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,4096,0.1476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,8192,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,16384,0.1648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,32768,0.1559
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,65536,0.1619
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1,1,1,131072,0.1745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,2,0.1353
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,4,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,8,0.1354
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,16,0.1353
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,32,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,64,0.1349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,128,0.1352
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,256,0.1349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,512,0.1377
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,1024,0.1392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,2048,0.1409
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,4096,0.1499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,8192,0.1569
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,16384,0.1496
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,32768,0.1575
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,65536,0.1689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,2,1,1,131072,0.1915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,2,0.1388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,4,0.1390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,8,0.1403
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,16,0.1393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,32,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,64,0.1386
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,128,0.1388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,256,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,512,0.1388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,1024,0.1463
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,2048,0.1544
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,4096,0.1593
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,8192,0.1524
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,16384,0.1570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,32768,0.1690
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,65536,0.1909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,4,1,1,131072,0.2402
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,2,0.1414
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,4,0.1412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,8,0.1396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,16,0.1419
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,32,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,64,0.1412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,128,0.1415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,256,0.1403
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,512,0.1449
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,1024,0.1537
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,2048,0.1616
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,4096,0.1487
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,8192,0.1549
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,16384,0.1696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,32768,0.1931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,65536,0.2399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,8,1,1,131072,0.3366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,2,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,4,0.1528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,8,0.1526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,16,0.1534
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,32,0.1526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,64,0.1532
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,128,0.1527
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,256,0.1526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,512,0.1571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,1024,0.1656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,2048,0.1576
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,4096,0.1649
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,8192,0.1783
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,16384,0.2040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,32768,0.2551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,65536,0.3588
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,16,1,1,131072,0.5614
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,2,0.1578
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,4,0.1570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,8,0.1573
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,16,0.1570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,32,0.1573
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,64,0.1573
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,128,0.1569
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,256,0.1587
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,512,0.1659
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,1024,0.1693
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,2048,0.1753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,4096,0.1896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,8192,0.2152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,16384,0.2680
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,32768,0.3700
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,65536,0.5732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,32,1,1,131072,0.9842
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,2,0.1757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,4,0.1764
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,8,0.1775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,16,0.1771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,32,0.1773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,64,0.1769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,128,0.1772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,256,0.1788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,512,0.1823
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,1024,0.1883
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,2048,0.2031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,4096,0.2288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,8192,0.2802
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,16384,0.3819
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,32768,0.5882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,65536,0.9940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,64,1,1,131072,1.8126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,2,0.2238
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,4,0.2229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,8,0.2225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,16,0.2235
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,32,0.2239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,64,0.2230
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,128,0.2239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,256,0.2268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,512,0.2340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,1024,0.2491
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,2048,0.2780
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,4096,0.3285
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,8192,0.4304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,16384,0.6344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,32768,1.0453
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,65536,1.8664
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,128,1,1,131072,3.5019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,2,0.3109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,4,0.3101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,8,0.3121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,16,0.3093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,32,0.3102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,64,0.3093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,128,0.3104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,256,0.3187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,512,0.3328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,1024,0.3620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,2048,0.4150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,4096,0.5217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,8192,0.7273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,16384,1.1371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,32768,1.9654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,65536,3.6258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,256,1,1,131072,6.9303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,2,0.4850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,4,0.4861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,8,0.4871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,16,0.4826
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,32,0.4849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,64,0.4855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,128,0.4877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,256,0.5035
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,512,0.5285
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,1024,0.5791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,2048,0.6800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,4096,0.8672
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,8192,1.2419
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,16384,1.9977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,32768,3.5034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,512,1,1,65536,6.5181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,2,0.8495
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,4,0.8475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,8,0.8494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,16,0.8490
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,32,0.8486
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,64,0.8497
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,128,0.8550
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,256,0.8769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,512,0.9264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,1024,1.0268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,2048,1.2248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,4096,1.6176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,8192,2.3555
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,16384,3.8601
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,128,1024,1,1,32768,6.8800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,2,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,4,0.1129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,8,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,16,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,32,0.1129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,64,0.1134
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,128,0.1155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,256,0.1151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,512,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,1024,0.1218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,2048,0.1215
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,4096,0.1233
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,8192,0.1290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,16384,0.1341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,32768,0.1443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,65536,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1,1,1,131072,0.1592
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,2,0.1151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,4,0.1174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,8,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,16,0.1162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,32,0.1159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,64,0.1153
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,128,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,256,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,512,0.1184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,1024,0.1214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,2048,0.1206
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,4096,0.1259
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,8192,0.1318
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,16384,0.1412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,32768,0.1449
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,65536,0.1602
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,2,1,1,131072,0.1841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,2,0.1199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,4,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,8,0.1197
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,16,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,32,0.1198
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,64,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,128,0.1193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,256,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,512,0.1218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,1024,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,2048,0.1273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,4096,0.1359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,8192,0.1433
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,16384,0.1443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,32768,0.1579
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,65536,0.1824
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,4,1,1,131072,0.2351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,2,0.1240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,4,0.1232
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,8,0.1229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,16,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,32,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,64,0.1232
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,128,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,256,0.1231
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,512,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,1024,0.1312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,2048,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,4096,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,8192,0.1441
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,16384,0.1579
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,32768,0.1835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,65536,0.2355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,8,1,1,131072,0.3531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,2,0.1309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,4,0.1309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,8,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,16,0.1316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,32,0.1320
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,64,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,128,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,256,0.1311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,512,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,1024,0.1434
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,2048,0.1511
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,4096,0.1490
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,8192,0.1637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,16384,0.1890
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,32768,0.2374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,65536,0.3373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,16,1,1,131072,0.5435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,2,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,4,0.1393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,8,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,16,0.1398
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,32,0.1395
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,64,0.1402
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,128,0.1393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,256,0.1402
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,512,0.1449
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,1024,0.1537
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,2048,0.1525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,4096,0.1688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,8192,0.1936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,16384,0.2489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,32768,0.3505
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,65536,0.5542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,32,1,1,131072,0.9746
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,2,0.1442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,4,0.1434
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,8,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,16,0.1434
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,32,0.1438
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,64,0.1444
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,128,0.1435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,256,0.1458
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,512,0.1565
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,1024,0.1638
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,2048,0.1798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,4096,0.2064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,8192,0.2589
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,16384,0.3618
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,32768,0.5649
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,65536,0.9822
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,64,1,1,131072,1.7934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,2,0.1670
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,4,0.1672
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,8,0.1677
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,16,0.1675
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,32,0.1677
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,64,0.1686
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,128,0.1686
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,256,0.1726
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,512,0.1797
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,1024,0.1966
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,2048,0.2228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,4096,0.2754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,8192,0.3758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,16384,0.5841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,32768,0.9962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,65536,1.8045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,128,1,1,131072,3.5224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,2,0.2139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,4,0.2158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,8,0.2161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,16,0.2155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,32,0.2144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,64,0.2140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,128,0.2161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,256,0.2262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,512,0.2410
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,1024,0.2689
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,2048,0.3244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,4096,0.4297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,8192,0.6365
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,16384,1.0562
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,32768,1.9049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,65536,3.5515
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,256,1,1,131072,7.0091
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,2,0.3072
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,4,0.3085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,8,0.3071
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,16,0.3086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,32,0.3070
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,64,0.3073
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,128,0.3136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,256,0.3277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,512,0.3594
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,1024,0.4137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,2048,0.5263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,4096,0.7389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,8192,1.1759
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,16384,2.0154
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,32768,3.7597
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,512,1,1,65536,7.1544
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,2,0.5090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,4,0.5068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,8,0.5097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,16,0.5082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,32,0.5069
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,64,0.5075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,128,0.5197
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,256,0.5430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,512,0.5935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,1024,0.6961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,2048,0.9068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,4096,1.3103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,8192,2.1033
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,16384,3.7218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,64,1024,1,1,32768,6.9404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,2,0.0985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,4,0.0988
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,8,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,16,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,32,0.0985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,64,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,128,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,256,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,512,0.1009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,1024,0.1046
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,2048,0.1048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,4096,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,8192,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,16384,0.1208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,32768,0.1271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,65536,0.1396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1,1,1,131072,0.1643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,2,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,4,0.0985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,8,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,16,0.0987
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,32,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,64,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,128,0.0993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,256,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,512,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,1024,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,2048,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,4096,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,8192,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,16384,0.1261
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,32768,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,65536,0.1510
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,2,1,1,131072,0.1847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,2,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,4,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,8,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,16,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,32,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,64,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,128,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,256,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,512,0.1073
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,1024,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,2048,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,4096,0.1176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,8192,0.1252
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,16384,0.1376
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,32768,0.1522
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,65536,0.1855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,4,1,1,131072,0.2545
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,2,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,4,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,8,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,16,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,32,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,64,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,128,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,256,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,512,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,1024,0.1185
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,2048,0.1173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,4096,0.1291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,8192,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,16384,0.1575
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,32768,0.1950
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,65536,0.2631
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,8,1,1,131072,0.3993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,2,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,4,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,8,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,16,0.1117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,32,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,64,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,128,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,256,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,512,0.1178
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,1024,0.1226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,2048,0.1314
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,4096,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,8192,0.1615
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,16384,0.1992
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,32768,0.2730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,65536,0.4265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,16,1,1,131072,0.7283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,2,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,4,0.1206
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,8,0.1199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,16,0.1171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,32,0.1207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,64,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,128,0.1190
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,256,0.1218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,512,0.1272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,1024,0.1377
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,2048,0.1500
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,4096,0.1690
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,8192,0.2048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,16384,0.2811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,32768,0.4312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,65536,0.7324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,32,1,1,131072,1.3353
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,2,0.1281
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,4,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,8,0.1299
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,16,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,32,0.1291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,64,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,128,0.1290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,256,0.1322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,512,0.1387
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,1024,0.1508
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,2048,0.1684
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,4096,0.2060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,8192,0.2798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,16384,0.4306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,32768,0.7264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,65536,1.3223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,64,1,1,131072,2.5187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,2,0.1504
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,4,0.1491
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,8,0.1511
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,16,0.1500
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,32,0.1495
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,64,0.1502
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,128,0.1499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,256,0.1523
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,512,0.1637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,1024,0.1826
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,2048,0.2211
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,4096,0.2978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,8192,0.4456
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,16384,0.7487
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,32768,1.3509
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,65536,2.5738
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,128,1,1,131072,5.0671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,2,0.1834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,4,0.1836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,8,0.1828
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,16,0.1833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,32,0.1837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,64,0.1842
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,128,0.1839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,256,0.1867
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,512,0.2099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,1024,0.2477
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,2048,0.3258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,4096,0.4734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,8192,0.7712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,16384,1.3754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,32768,2.6594
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,65536,5.2475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,256,1,1,131072,10.4841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,2,0.2450
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,4,0.2448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,8,0.2444
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,16,0.2452
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,32,0.2451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,64,0.2453
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,128,0.2481
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,256,0.2528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,512,0.2909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,1024,0.3600
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,2048,0.4994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,4096,0.7599
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,8192,1.2930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,16384,2.4384
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,32768,4.8948
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,512,1,1,65536,9.7515
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,2,0.3743
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,4,0.3747
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,8,0.3747
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,16,0.3739
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,32,0.3760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,64,0.3744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,128,0.3816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,256,0.3909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,512,0.4630
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,1024,0.6018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,2048,0.8744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,4096,1.4002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,8192,2.4651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,16384,4.7925
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,32,1024,1,1,32768,9.8447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,2,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,4,0.0909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,8,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,16,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,32,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,64,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,128,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,256,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,512,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,1024,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,2048,0.0973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,4096,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,8192,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,16384,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,32768,0.1184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,65536,0.1288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1,1,1,131072,0.1413
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,2,0.0929
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,4,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,8,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,16,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,32,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,64,0.0929
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,128,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,256,0.0929
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,512,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,1024,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,2048,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,4096,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,8192,0.1030
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,16384,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,32768,0.1292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,65536,0.1425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,2,1,1,131072,0.1682
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,2,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,4,0.0932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,8,0.0927
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,16,0.0932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,32,0.0930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,64,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,128,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,256,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,512,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,1024,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,2048,0.0966
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,4096,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,8192,0.1087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,16384,0.1270
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,32768,0.1378
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,65536,0.1630
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,4,1,1,131072,0.2124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,2,0.0985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,4,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,8,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,16,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,32,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,64,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,128,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,256,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,512,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,1024,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,2048,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,4096,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,8192,0.1288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,16384,0.1416
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,32768,0.1660
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,65536,0.2163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,8,1,1,131072,0.3156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,2,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,4,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,8,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,16,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,32,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,64,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,128,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,256,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,512,0.1089
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,1024,0.1130
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,2048,0.1189
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,4096,0.1338
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,8192,0.1463
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,16384,0.1737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,32768,0.2229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,65536,0.3194
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,16,1,1,131072,0.5181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,2,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,4,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,8,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,16,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,32,0.1087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,64,0.1086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,128,0.1088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,256,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,512,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,1024,0.1210
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,2048,0.1356
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,4096,0.1497
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,8192,0.1758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,16384,0.2265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,32768,0.3306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,65536,0.5332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,32,1,1,131072,0.9338
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,2,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,4,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,8,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,16,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,32,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,64,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,128,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,256,0.1158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,512,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,1024,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,2048,0.1550
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,4096,0.1791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,8192,0.2319
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,16384,0.3337
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,32768,0.5390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,65536,0.9367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,64,1,1,131072,1.7292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,2,0.1278
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,4,0.1240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,8,0.1250
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,16,0.1247
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,32,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,64,0.1245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,128,0.1257
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,256,0.1284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,512,0.1443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,1024,0.1567
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,2048,0.1825
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,4096,0.2323
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,8192,0.3335
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,16384,0.5359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,32768,0.9364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,65536,1.7262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,128,1,1,131072,3.3684
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,2,0.1451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,4,0.1494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,8,0.1451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,16,0.1496
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,32,0.1463
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,64,0.1504
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,128,0.1516
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,256,0.1565
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,512,0.1731
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,1024,0.1980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,2048,0.2495
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,4096,0.3523
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,8192,0.5539
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,16384,0.9637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,32768,1.7833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,65536,3.3874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,256,1,1,131072,6.7217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,2,0.1877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,4,0.1876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,8,0.1873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,16,0.1872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,32,0.1877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,64,0.1881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,128,0.1917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,256,0.1980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,512,0.2286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,1024,0.2811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,2048,0.3881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,4096,0.5908
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,8192,1.0093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,16384,1.8146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,32768,3.4929
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,512,1,1,65536,6.7599
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,2,0.2629
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,4,0.2623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,8,0.2628
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,16,0.2627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,32,0.2629
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,64,0.2622
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,128,0.2699
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,256,0.2824
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,512,0.3375
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,1024,0.4326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,2048,0.6343
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,4096,1.0248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,8192,1.8010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,16384,3.3574
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,16,1024,1,1,32768,6.4677
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,2,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,4,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,8,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,16,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,32,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,64,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,128,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,256,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,512,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,1024,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,2048,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,4096,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,8192,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,16384,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,32768,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,65536,0.1253
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1,1,1,131072,0.1380
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,2,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,4,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,8,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,16,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,32,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,64,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,128,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,256,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,512,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,1024,0.0945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,2048,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,4096,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,8192,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,16384,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,32768,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,65536,0.1393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,2,1,1,131072,0.1652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,2,0.0932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,4,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,8,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,16,0.0925
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,32,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,64,0.0931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,128,0.0932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,256,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,512,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,1024,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,2048,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,4096,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,8192,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,16384,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,32768,0.1376
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,65536,0.1610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,4,1,1,131072,0.2081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,2,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,4,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,8,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,16,0.0965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,32,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,64,0.0965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,128,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,256,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,512,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,1024,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,2048,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,4096,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,8192,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,16384,0.1389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,32768,0.1619
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,65536,0.2125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,8,1,1,131072,0.3112
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,2,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,4,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,8,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,16,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,32,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,64,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,128,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,256,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,512,0.1047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,1024,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,2048,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,4096,0.1265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,8192,0.1423
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,16384,0.1700
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,32768,0.2180
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,65536,0.3174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,16,1,1,131072,0.5125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,2,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,4,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,8,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,16,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,32,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,64,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,128,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,256,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,512,0.1086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,1024,0.1130
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,2048,0.1276
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,4096,0.1452
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,8192,0.1694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,16384,0.2197
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,32768,0.3217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,65536,0.5249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,32,1,1,131072,0.9298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,2,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,4,0.1096
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,8,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,16,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,32,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,64,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,128,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,256,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,512,0.1151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,1024,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,2048,0.1466
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,4096,0.1732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,8192,0.2231
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,16384,0.3242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,32768,0.5290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,65536,0.9330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,64,1,1,131072,1.7182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,2,0.1156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,4,0.1167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,8,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,16,0.1176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,32,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,64,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,128,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,256,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,512,0.1322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,1024,0.1493
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,2048,0.1756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,4096,0.2257
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,8192,0.3257
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,16384,0.5270
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,32768,0.9276
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,65536,1.7094
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,128,1,1,131072,3.3572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,2,0.1323
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,4,0.1325
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,8,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,16,0.1326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,32,0.1326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,64,0.1322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,128,0.1378
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,256,0.1448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,512,0.1596
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,1024,0.1847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,2048,0.2374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,4096,0.3399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,8192,0.5374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,16384,0.9411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,32768,1.7720
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,65536,3.3582
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,256,1,1,131072,6.7147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,2,0.1640
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,4,0.1632
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,8,0.1642
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,16,0.1651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,32,0.1658
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,64,0.1619
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,128,0.1703
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,256,0.1763
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,512,0.2037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,1024,0.2570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,2048,0.3603
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,4096,0.5657
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,8192,0.9796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,16384,1.7909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,32768,3.4586
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,512,1,1,65536,6.7147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,2,0.2196
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,4,0.2183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,8,0.2201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,16,0.2205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,32,0.2200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,64,0.2204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,128,0.2256
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,256,0.2397
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,512,0.2903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,1024,0.3836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,2048,0.5880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,4096,0.9765
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,8192,1.7530
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,16384,3.3074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,8,1024,1,1,32768,6.3980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,2,0.0869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,4,0.0828
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,8,0.0830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,16,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,32,0.0830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,64,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,128,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,256,0.0822
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,512,0.0847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,1024,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,2048,0.0863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,4096,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,8192,0.0884
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,16384,0.0923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,32768,0.1045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,65536,0.1195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1,1,1,131072,0.1328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,2,0.0891
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,4,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,8,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,16,0.0889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,32,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,64,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,128,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,256,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,512,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,1024,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,2048,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,4096,0.0926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,8192,0.0965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,16384,0.1029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,32768,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,65536,0.1410
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,2,1,1,131072,0.1615
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,2,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,4,0.0923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,8,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,16,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,32,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,64,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,128,0.0882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,256,0.0882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,512,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,1024,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,2048,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,4096,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,8192,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,16384,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,32768,0.1337
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,65536,0.1578
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,4,1,1,131072,0.2064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,2,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,4,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,8,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,16,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,32,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,64,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,128,0.0944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,256,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,512,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,1024,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,2048,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,4096,0.1028
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,8192,0.1168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,16384,0.1349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,32768,0.1583
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,65536,0.2095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,8,1,1,131072,0.3099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,2,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,4,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,8,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,16,0.0988
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,32,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,64,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,128,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,256,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,512,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,1024,0.1047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,2048,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,4096,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,8192,0.1404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,16384,0.1653
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,32768,0.2141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,65536,0.3133
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,16,1,1,131072,0.5100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,2,0.1050
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,4,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,8,0.1047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,16,0.1044
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,32,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,64,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,128,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,256,0.1049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,512,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,1024,0.1106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,2048,0.1241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,4096,0.1436
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,8192,0.1682
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,16384,0.2187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,32768,0.3236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,65536,0.5239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,32,1,1,131072,0.9265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,2,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,4,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,8,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,16,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,32,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,64,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,128,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,256,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,512,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,1024,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,2048,0.1447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,4096,0.1706
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,8192,0.2224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,16384,0.3228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,32768,0.5239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,65536,0.9289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,64,1,1,131072,1.7192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,2,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,4,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,8,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,16,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,32,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,64,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,128,0.1132
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,256,0.1162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,512,0.1286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,1024,0.1459
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,2048,0.1711
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,4096,0.2225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,8192,0.3244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,16384,0.5244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,32768,0.9239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,65536,1.7083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,128,1,1,131072,3.3528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,2,0.1249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,4,0.1280
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,8,0.1261
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,16,0.1271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,32,0.1272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,64,0.1283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,128,0.1294
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,256,0.1389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,512,0.1532
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,1024,0.1798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,2048,0.2334
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,4096,0.3348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,8192,0.5328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,16384,0.9316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,32768,1.7595
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,65536,3.3625
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,256,1,1,131072,6.6921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,2,0.1570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,4,0.1551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,8,0.1552
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,16,0.1549
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,32,0.1558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,64,0.1551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,128,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,256,0.1671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,512,0.1941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,1024,0.2445
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,2048,0.3516
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,4096,0.5571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,8192,0.9691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,16384,1.7785
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,32768,3.4392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,512,1,1,65536,6.6902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,2,0.1993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,4,0.1999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,8,0.2008
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,16,0.1999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,32,0.2009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,64,0.2003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,128,0.2067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,256,0.2187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,512,0.2678
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,1024,0.3623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,2048,0.5652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,4096,0.9494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,8192,1.7166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,16384,3.2699
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,4,1024,1,1,32768,6.3748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,2,0.0821
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,4,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,8,0.0821
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,16,0.0823
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,32,0.0831
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,64,0.0823
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,128,0.0824
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,256,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,512,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,1024,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,2048,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,4096,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,8192,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,16384,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,32768,0.1049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,65536,0.1214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1,1,1,131072,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,2,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,4,0.0850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,8,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,16,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,32,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,64,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,128,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,256,0.0826
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,512,0.0842
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,1024,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,2048,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,4096,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,8192,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,16384,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,32768,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,65536,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,2,1,1,131072,0.1634
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,2,0.0902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,4,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,8,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,16,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,32,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,64,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,128,0.0904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,256,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,512,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,1024,0.0909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,2048,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,4096,0.0967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,8192,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,16384,0.1154
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,32768,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,65536,0.1580
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,4,1,1,131072,0.2083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,2,0.0944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,4,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,8,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,16,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,32,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,64,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,128,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,256,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,512,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,1024,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,2048,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,4096,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,8192,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,16384,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,32768,0.1587
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,65536,0.2059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,8,1,1,131072,0.3248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,2,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,4,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,8,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,16,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,32,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,64,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,128,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,256,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,512,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,1024,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,2048,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,4096,0.1203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,8192,0.1387
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,16384,0.1655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,32768,0.2131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,65536,0.3155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,16,1,1,131072,0.5093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,2,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,4,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,8,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,16,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,32,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,64,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,128,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,256,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,512,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,1024,0.1071
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,2048,0.1210
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,4096,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,8192,0.1651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,16384,0.2141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,32768,0.3183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,65536,0.5190
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,32,1,1,131072,0.9268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,2,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,4,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,8,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,16,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,32,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,64,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,128,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,256,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,512,0.1088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,1024,0.1236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,2048,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,4096,0.1682
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,8192,0.2206
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,16384,0.3200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,32768,0.5234
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,65536,0.9291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,64,1,1,131072,1.7123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,2,0.1114
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,4,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,8,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,16,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,32,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,64,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,128,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,256,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,512,0.1254
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,1024,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,2048,0.1700
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,4096,0.2212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,8192,0.3226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,16384,0.5252
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,32768,0.9220
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,65536,1.7076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,128,1,1,131072,3.3445
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,2,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,4,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,8,0.1220
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,16,0.1231
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,32,0.1234
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,64,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,128,0.1263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,256,0.1349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,512,0.1503
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,1024,0.1749
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,2048,0.2302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,4096,0.3313
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,8192,0.5288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,16384,0.9316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,32768,1.7520
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,65536,3.3464
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,256,1,1,131072,6.5989
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,2,0.1506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,4,0.1488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,8,0.1508
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,16,0.1497
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,32,0.1510
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,64,0.1507
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,128,0.1542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,256,0.1618
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,512,0.1881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,1024,0.2394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,2048,0.3445
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,4096,0.5476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,8192,0.9651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,16384,1.7704
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,32768,3.4365
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,512,1,1,65536,6.7013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,2,0.1920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,4,0.1919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,8,0.1915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,16,0.1916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,32,0.1913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,64,0.1919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,128,0.1979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,256,0.2100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,512,0.2591
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,1024,0.3512
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,2048,0.5537
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,4096,0.9385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,8192,1.7264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,16384,3.2636
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,2,1024,1,1,32768,6.3742
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,2,0.0808
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,4,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,8,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,16,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,32,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,64,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,128,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,256,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,512,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,1024,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,2048,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,4096,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,8192,0.0863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,16384,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,32768,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,65536,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1,1,1,131072,0.1319
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,2,0.0850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,4,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,8,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,16,0.0847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,32,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,64,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,128,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,256,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,512,0.0862
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,1024,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,2048,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,4096,0.0881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,8192,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,16384,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,32768,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,65536,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,2,1,1,131072,0.1604
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,2,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,4,0.0840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,8,0.0841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,16,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,32,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,64,0.0839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,128,0.0841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,256,0.0845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,512,0.0862
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,1024,0.0872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,2048,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,4096,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,8192,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,16384,0.1109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,32768,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,65536,0.1547
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,4,1,1,131072,0.2049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,2,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,4,0.0912
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,8,0.0906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,16,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,32,0.0906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,64,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,128,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,256,0.0912
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,512,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,1024,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,2048,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,4096,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,8192,0.1117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,16384,0.1311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,32768,0.1560
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,65536,0.2047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,8,1,1,131072,0.3226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,2,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,4,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,8,0.0969
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,16,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,32,0.0966
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,64,0.0969
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,128,0.0972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,256,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,512,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,1024,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,2048,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,4096,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,8192,0.1357
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,16384,0.1630
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,32768,0.2113
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,65536,0.3101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,16,1,1,131072,0.5070
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,2,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,4,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,8,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,16,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,32,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,64,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,128,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,256,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,512,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,1024,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,2048,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,4096,0.1393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,8192,0.1619
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,16384,0.2128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,32768,0.3177
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,65536,0.5150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,32,1,1,131072,0.9189
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,2,0.1013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,4,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,8,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,16,0.1014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,32,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,64,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,128,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,256,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,512,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,1024,0.1195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,2048,0.1384
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,4096,0.1648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,8192,0.2163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,16384,0.3167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,32768,0.5210
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,65536,0.9234
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,64,1,1,131072,1.7108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,2,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,4,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,8,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,16,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,32,0.1086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,64,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,128,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,256,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,512,0.1208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,1024,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,2048,0.1668
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,4096,0.2176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,8192,0.3201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,16384,0.5202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,32768,0.9205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,65536,1.7007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,128,1,1,131072,3.3428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,2,0.1194
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,4,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,8,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,16,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,32,0.1191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,64,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,128,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,256,0.1313
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,512,0.1477
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,1024,0.1723
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,2048,0.2271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,4096,0.3278
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,8192,0.5270
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,16384,0.9299
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,32768,1.7521
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,65536,3.3551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,256,1,1,131072,6.6866
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,2,0.1446
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,4,0.1448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,8,0.1447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,16,0.1446
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,32,0.1452
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,64,0.1450
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,128,0.1510
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,256,0.1571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,512,0.1852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,1024,0.2348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,2048,0.3407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,4096,0.5459
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,8192,0.9557
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,16384,1.7649
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,32768,3.4290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,512,1,1,65536,6.7080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,2,0.1853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,4,0.1841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,8,0.1842
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,16,0.1851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,32,0.1854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,64,0.1844
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,128,0.1910
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,256,0.2034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,512,0.2518
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,1024,0.3441
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,2048,0.5475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,4096,0.9364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,8192,1.7101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,16384,3.2648
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,fp8_block,1,1024,1,1,32768,6.3615
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,2,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,4,0.1317
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,8,0.1332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,16,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,32,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,64,0.1327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,128,0.1321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,256,0.1312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,512,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,1024,0.1369
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,2048,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,4096,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,8192,0.1419
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,16384,0.1473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,32768,0.1523
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,65536,0.1553
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1,1,1,131072,0.1621
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,2,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,4,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,8,0.1312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,16,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,32,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,64,0.1319
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,128,0.1312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,256,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,512,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,1024,0.1358
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,2048,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,4096,0.1389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,8192,0.1437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,16384,0.1455
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,32768,0.1495
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,65536,0.1542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,2,1,1,131072,0.1670
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,2,0.1355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,4,0.1356
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,8,0.1342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,16,0.1352
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,32,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,64,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,128,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,256,0.1341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,512,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,1024,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,2048,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,4096,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,8192,0.1449
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,16384,0.1476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,32768,0.1542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,65536,0.1658
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,4,1,1,131072,0.1898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,2,0.1332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,4,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,8,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,16,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,32,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,64,0.1340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,128,0.1330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,256,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,512,0.1346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,1024,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,2048,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,4096,0.1435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,8192,0.1464
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,16384,0.1539
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,32768,0.1656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,65536,0.1894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,8,1,1,131072,0.2351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,2,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,4,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,8,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,16,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,32,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,64,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,128,0.1426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,256,0.1425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,512,0.1442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,1024,0.1495
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,2048,0.1507
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,4096,0.1539
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,8192,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,16384,0.1734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,32768,0.1991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,65536,0.2500
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,16,1,1,131072,0.3506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,2,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,4,0.1471
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,8,0.1474
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,16,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,32,0.1466
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,64,0.1472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,128,0.1466
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,256,0.1479
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,512,0.1550
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,1024,0.1568
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,2048,0.1598
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,4096,0.1665
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,8192,0.1792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,16384,0.2062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,32768,0.2566
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,65536,0.3577
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,32,1,1,131072,0.5627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,2,0.1567
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,4,0.1586
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,8,0.1578
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,16,0.1584
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,32,0.1571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,64,0.1574
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,128,0.1570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,256,0.1585
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,512,0.1610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,1024,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,2048,0.1705
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,4096,0.1840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,8192,0.2094
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,16384,0.2604
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,32768,0.3624
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,65536,0.5694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,64,1,1,131072,0.9767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,2,0.1861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,4,0.1858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,8,0.1856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,16,0.1859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,32,0.1861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,64,0.1863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,128,0.1853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,256,0.1879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,512,0.1904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,1024,0.1977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,2048,0.2117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,4096,0.2391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,8192,0.2899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,16384,0.3932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,32768,0.6035
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,65536,1.0138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,128,1,1,131072,1.8342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,2,0.2391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,4,0.2393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,8,0.2374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,16,0.2371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,32,0.2377
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,64,0.2388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,128,0.2373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,256,0.2405
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,512,0.2478
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,1024,0.2618
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,2048,0.2893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,4096,0.3432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,8192,0.4465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,16384,0.6591
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,32768,1.0888
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,65536,1.9008
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,256,1,1,131072,3.5417
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,2,0.3336
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,4,0.3313
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,8,0.3324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,16,0.3325
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,32,0.3327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,64,0.3322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,128,0.3322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,256,0.3359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,512,0.3496
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,1024,0.3770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,2048,0.4279
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,4096,0.5260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,8192,0.7132
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,16384,1.0862
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,32768,1.8236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,512,1,1,65536,3.3030
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,2,0.5505
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,4,0.5508
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,8,0.5506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,16,0.5506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,32,0.5506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,64,0.5500
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,128,0.5505
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,256,0.5582
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,512,0.5845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,1024,0.6354
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,2048,0.7339
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,4096,0.9290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,8192,1.2993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,16384,2.0396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,128,1024,1,1,32768,3.5217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,2,0.1129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,4,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,8,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,16,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,32,0.1112
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,64,0.1130
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,128,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,256,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,512,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,1024,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,2048,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,4096,0.1169
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,8192,0.1198
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,16384,0.1230
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,32768,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,65536,0.1327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1,1,1,131072,0.1409
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,2,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,4,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,8,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,16,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,32,0.1131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,64,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,128,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,256,0.1131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,512,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,1024,0.1170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,2048,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,4096,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,8192,0.1233
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,16384,0.1286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,32768,0.1373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,65536,0.1414
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,2,1,1,131072,0.1539
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,2,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,4,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,8,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,16,0.1171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,32,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,64,0.1152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,128,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,256,0.1168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,512,0.1179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,1024,0.1199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,2048,0.1207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,4096,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,8192,0.1276
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,16384,0.1341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,32768,0.1390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,65536,0.1503
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,4,1,1,131072,0.1732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,2,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,4,0.1193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,8,0.1192
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,16,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,32,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,64,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,128,0.1191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,256,0.1191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,512,0.1199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,1024,0.1251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,2048,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,4096,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,8192,0.1336
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,16384,0.1387
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,32768,0.1502
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,65536,0.1732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,8,1,1,131072,0.2165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,2,0.1238
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,4,0.1237
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,8,0.1236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,16,0.1232
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,32,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,64,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,128,0.1218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,256,0.1232
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,512,0.1251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,1024,0.1290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,2048,0.1342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,4096,0.1364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,8192,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,16384,0.1536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,32768,0.1772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,65536,0.2195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,16,1,1,131072,0.3100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,2,0.1289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,4,0.1295
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,8,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,16,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,32,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,64,0.1306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,128,0.1286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,256,0.1296
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,512,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,1024,0.1357
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,2048,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,4096,0.1443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,8192,0.1569
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,16384,0.1816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,32768,0.2297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,65536,0.3229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,32,1,1,131072,0.5154
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,2,0.1332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,4,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,8,0.1330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,16,0.1333
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,32,0.1333
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,64,0.1337
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,128,0.1337
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,256,0.1342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,512,0.1426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,1024,0.1453
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,2048,0.1523
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,4096,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,8192,0.1882
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,16384,0.2361
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,32768,0.3301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,65536,0.5241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,64,1,1,131072,0.8994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,2,0.1507
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,4,0.1509
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,8,0.1496
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,16,0.1509
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,32,0.1506
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,64,0.1505
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,128,0.1499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,256,0.1523
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,512,0.1556
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,1024,0.1627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,2048,0.1745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,4096,0.1981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,8192,0.2462
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,16384,0.3399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,32768,0.5373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,65536,0.9156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,128,1,1,131072,1.6627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,2,0.1813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,4,0.1812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,8,0.1815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,16,0.1804
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,32,0.1821
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,64,0.1813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,128,0.1824
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,256,0.1845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,512,0.1920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,1024,0.2054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,2048,0.2314
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,4096,0.2799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,8192,0.3777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,16384,0.5738
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,32768,0.9581
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,65536,1.7132
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,256,1,1,131072,3.2262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,2,0.2453
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,4,0.2469
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,8,0.2472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,16,0.2468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,32,0.2462
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,64,0.2468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,128,0.2464
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,256,0.2531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,512,0.2661
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,1024,0.2946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,2048,0.3447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,4096,0.4439
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,8192,0.6443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,16384,1.0383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,32768,1.8090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,512,1,1,65536,3.3499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,2,0.3736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,4,0.3746
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,8,0.3747
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,16,0.3757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,32,0.3753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,64,0.3750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,128,0.3775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,256,0.3894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,512,0.4139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,1024,0.4631
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,2048,0.5571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,4096,0.7376
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,8192,1.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,16384,1.7981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,64,1024,1,1,32768,3.2094
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,2,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,4,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,8,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,16,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,32,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,64,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,128,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,256,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,512,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,1024,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,2048,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,4096,0.1109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,8192,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,16384,0.1193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,32768,0.1246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,65536,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1,1,1,131072,0.1423
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,2,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,4,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,8,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,16,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,32,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,64,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,128,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,256,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,512,0.1086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,1024,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,2048,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,4096,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,8192,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,16384,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,32768,0.1289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,65536,0.1370
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,2,1,1,131072,0.1535
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,2,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,4,0.1095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,8,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,16,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,32,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,64,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,128,0.1088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,256,0.1096
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,512,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,1024,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,2048,0.1132
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,4096,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,8192,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,16384,0.1319
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,32768,0.1392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,65536,0.1534
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,4,1,1,131072,0.1836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,2,0.1145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,4,0.1154
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,8,0.1158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,16,0.1155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,32,0.1156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,64,0.1149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,128,0.1145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,256,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,512,0.1154
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,1024,0.1188
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,2048,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,4096,0.1279
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,8192,0.1351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,16384,0.1448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,32768,0.1621
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,65536,0.1914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,8,1,1,131072,0.2519
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,2,0.1213
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,4,0.1208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,8,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,16,0.1218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,32,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,64,0.1209
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,128,0.1219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,256,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,512,0.1236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,1024,0.1275
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,2048,0.1311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,4096,0.1370
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,8192,0.1462
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,16384,0.1662
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,32768,0.1988
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,65536,0.2675
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,16,1,1,131072,0.4029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,2,0.1263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,4,0.1275
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,8,0.1268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,16,0.1286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,32,0.1268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,64,0.1279
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,128,0.1267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,256,0.1282
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,512,0.1300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,1024,0.1364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,2048,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,4096,0.1529
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,8192,0.1700
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,16384,0.2043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,32768,0.2724
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,65536,0.4074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,32,1,1,131072,0.6792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,2,0.1377
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,4,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,8,0.1384
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,16,0.1377
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,32,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,64,0.1375
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,128,0.1372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,256,0.1392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,512,0.1421
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,1024,0.1476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,2048,0.1577
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,4096,0.1756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,8192,0.2091
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,16384,0.2773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,32768,0.4128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,65536,0.6847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,64,1,1,131072,1.2236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,2,0.1651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,4,0.1637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,8,0.1639
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,16,0.1630
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,32,0.1644
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,64,0.1637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,128,0.1646
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,256,0.1655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,512,0.1701
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,1024,0.1787
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,2048,0.1974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,4096,0.2332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,8192,0.3026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,16384,0.4385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,32768,0.7125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,65536,1.2629
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,128,1,1,131072,2.3820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,2,0.2066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,4,0.2066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,8,0.2074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,16,0.2084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,32,0.2067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,64,0.2073
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,128,0.2079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,256,0.2073
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,512,0.2167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,1024,0.2360
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,2048,0.2702
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,4096,0.3424
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,8192,0.4778
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,16384,0.7490
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,32768,1.2958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,65536,2.4283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,256,1,1,131072,4.7180
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,2,0.2875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,4,0.2867
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,8,0.2863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,16,0.2869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,32,0.2869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,64,0.2864
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,128,0.2865
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,256,0.2862
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,512,0.3031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,1024,0.3364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,2048,0.4022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,4096,0.5249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,8192,0.7635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,16384,1.2505
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,32768,2.2493
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,512,1,1,65536,4.3590
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,2,0.4735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,4,0.4728
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,8,0.4733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,16,0.4729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,32,0.4730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,64,0.4718
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,128,0.4729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,256,0.4721
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,512,0.5020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,1024,0.5652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,2048,0.6892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,4096,0.9371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,8192,1.4122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,16384,2.3763
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,32,1024,1,1,32768,4.3904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,2,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,4,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,8,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,16,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,32,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,64,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,128,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,256,0.0991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,512,0.1009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,1024,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,2048,0.1036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,4096,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,8192,0.1045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,16384,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,32768,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,65536,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1,1,1,131072,0.1309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,2,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,4,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,8,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,16,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,32,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,64,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,128,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,256,0.0993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,512,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,1024,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,2048,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,4096,0.1025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,8192,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,16384,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,32768,0.1168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,65536,0.1318
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,2,1,1,131072,0.1426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,2,0.1013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,4,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,8,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,16,0.1014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,32,0.1006
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,64,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,128,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,256,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,512,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,1024,0.1034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,2048,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,4096,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,8192,0.1097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,16384,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,32768,0.1263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,65536,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,4,1,1,131072,0.1615
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,2,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,4,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,8,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,16,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,32,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,64,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,128,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,256,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,512,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,1024,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,2048,0.1107
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,4096,0.1136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,8192,0.1198
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,16384,0.1309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,32768,0.1426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,65536,0.1633
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,8,1,1,131072,0.2070
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,2,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,4,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,8,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,16,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,32,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,64,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,128,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,256,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,512,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,1024,0.1155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,2048,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,4096,0.1245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,8192,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,16384,0.1488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,32768,0.1715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,65536,0.2119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,16,1,1,131072,0.2961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,2,0.1158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,4,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,8,0.1166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,16,0.1162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,32,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,64,0.1177
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,128,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,256,0.1171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,512,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,1024,0.1230
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,2048,0.1249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,4096,0.1372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,8192,0.1490
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,16384,0.1719
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,32768,0.2165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,65536,0.3035
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,32,1,1,131072,0.4816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,2,0.1226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,4,0.1207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,8,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,16,0.1210
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,32,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,64,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,128,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,256,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,512,0.1255
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,1024,0.1329
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,2048,0.1425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,4096,0.1553
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,8192,0.1769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,16384,0.2212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,32768,0.3085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,65536,0.4844
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,64,1,1,131072,0.8316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,2,0.1348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,4,0.1343
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,8,0.1343
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,16,0.1364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,32,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,64,0.1360
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,128,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,256,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,512,0.1444
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,1024,0.1513
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,2048,0.1633
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,4096,0.1861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,8192,0.2284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,16384,0.3163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,32768,0.4918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,65536,0.8383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,128,1,1,131072,1.5341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,2,0.1615
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,4,0.1610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,8,0.1614
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,16,0.1627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,32,0.1632
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,64,0.1622
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,128,0.1636
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,256,0.1666
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,512,0.1732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,1024,0.1843
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,2048,0.2095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,4096,0.2552
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,8192,0.3448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,16384,0.5191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,32768,0.8673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,65536,1.5692
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,256,1,1,131072,2.9729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,2,0.2090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,4,0.2101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,8,0.2103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,16,0.2106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,32,0.2109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,64,0.2096
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,128,0.2101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,256,0.2141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,512,0.2278
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,1024,0.2547
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,2048,0.2987
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,4096,0.3917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,8192,0.5699
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,16384,0.9287
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,32768,1.6435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,512,1,1,65536,3.0777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,2,0.3002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,4,0.3010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,8,0.3019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,16,0.3013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,32,0.3012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,64,0.3014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,128,0.3026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,256,0.3087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,512,0.3331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,1024,0.3775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,2048,0.4647
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,4096,0.6343
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,8192,0.9668
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,16384,1.6290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,16,1024,1,1,32768,2.9594
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,2,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,4,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,8,0.0949
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,16,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,32,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,64,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,128,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,256,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,512,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,1024,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,2048,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,4096,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,8192,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,16384,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,32768,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,65536,0.1145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1,1,1,131072,0.1251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,2,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,4,0.0993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,8,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,16,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,32,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,64,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,128,0.0992
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,256,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,512,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,1024,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,2048,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,4096,0.1026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,8192,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,16384,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,32768,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,65536,0.1268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,2,1,1,131072,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,2,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,4,0.1010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,8,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,16,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,32,0.1013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,64,0.1014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,128,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,256,0.1006
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,512,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,1024,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,2048,0.1033
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,4096,0.1055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,8192,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,16384,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,32768,0.1214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,65536,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,4,1,1,131072,0.1577
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,2,0.1045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,4,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,8,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,16,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,32,0.1044
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,64,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,128,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,256,0.1044
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,512,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,1024,0.1066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,2048,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,4096,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,8192,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,16384,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,32768,0.1390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,65536,0.1597
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,8,1,1,131072,0.2007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,2,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,4,0.1108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,8,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,16,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,32,0.1106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,64,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,128,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,256,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,512,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,1024,0.1136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,2048,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,4096,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,8192,0.1322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,16384,0.1435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,32768,0.1674
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,65536,0.2077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,16,1,1,131072,0.2904
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,2,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,4,0.1131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,8,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,16,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,32,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,64,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,128,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,256,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,512,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,1024,0.1166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,2048,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,4096,0.1320
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,8192,0.1438
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,16384,0.1654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,32768,0.2110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,65536,0.2979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,32,1,1,131072,0.4730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,2,0.1168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,4,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,8,0.1153
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,16,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,32,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,64,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,128,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,256,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,512,0.1166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,1024,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,2048,0.1357
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,4096,0.1465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,8192,0.1697
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,16384,0.2147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,32768,0.3025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,65536,0.4764
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,64,1,1,131072,0.8231
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,2,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,4,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,8,0.1233
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,16,0.1226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,32,0.1226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,64,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,128,0.1236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,256,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,512,0.1297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,1024,0.1388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,2048,0.1511
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,4096,0.1747
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,8192,0.2191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,16384,0.3053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,32768,0.4807
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,65536,0.8289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,128,1,1,131072,1.5217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,2,0.1407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,4,0.1412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,8,0.1409
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,16,0.1401
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,32,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,64,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,128,0.1418
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,256,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,512,0.1535
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,1024,0.1653
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,2048,0.1899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,4096,0.2345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,8192,0.3222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,16384,0.4977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,32768,0.8489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,65536,1.5488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,256,1,1,131072,2.9558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,2,0.1734
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,4,0.1737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,8,0.1726
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,16,0.1736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,32,0.1737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,64,0.1726
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,128,0.1741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,256,0.1778
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,512,0.1888
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,1024,0.2145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,2048,0.2599
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,4096,0.3515
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,8192,0.5300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,16384,0.8876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,32768,1.6023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,512,1,1,65536,3.0355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,2,0.2347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,4,0.2331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,8,0.2340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,16,0.2348
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,32,0.2345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,64,0.2333
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,128,0.2347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,256,0.2409
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,512,0.2661
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,1024,0.3091
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,2048,0.3945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,4096,0.5624
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,8192,0.8974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,16384,1.5599
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,8,1024,1,1,32768,2.8864
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,2,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,4,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,8,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,16,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,32,0.0926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,64,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,128,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,256,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,512,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,1024,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,2048,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,4096,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,8192,0.0973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,16384,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,32768,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,65536,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1,1,1,131072,0.1237
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,2,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,4,0.0930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,8,0.0925
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,16,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,32,0.0931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,64,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,128,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,256,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,512,0.0952
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,1024,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,2048,0.0945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,4096,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,8192,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,16384,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,32768,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,65536,0.1208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,2,1,1,131072,0.1346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,2,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,4,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,8,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,16,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,32,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,64,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,128,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,256,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,512,0.0967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,1024,0.0970
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,2048,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,4096,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,8192,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,16384,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,32768,0.1203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,65536,0.1334
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,4,1,1,131072,0.1564
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,2,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,4,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,8,0.1014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,16,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,32,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,64,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,128,0.1006
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,256,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,512,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,1024,0.1027
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,2048,0.1047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,4096,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,8192,0.1090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,16384,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,32768,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,65536,0.1572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,8,1,1,131072,0.1993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,2,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,4,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,8,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,16,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,32,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,64,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,128,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,256,0.1066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,512,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,1024,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,2048,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,4096,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,8192,0.1246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,16384,0.1392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,32768,0.1612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,65536,0.2039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,16,1,1,131072,0.2865
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,2,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,4,0.1119
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,8,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,16,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,32,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,64,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,128,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,256,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,512,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,1024,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,2048,0.1185
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,4096,0.1286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,8192,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,16384,0.1651
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,32768,0.2084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,65536,0.2968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,32,1,1,131072,0.4707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,2,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,4,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,8,0.1155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,16,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,32,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,64,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,128,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,256,0.1134
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,512,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,1024,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,2048,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,4096,0.1455
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,8192,0.1669
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,16384,0.2106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,32768,0.2984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,65536,0.4733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,64,1,1,131072,0.8210
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,2,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,4,0.1185
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,8,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,16,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,32,0.1178
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,64,0.1185
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,128,0.1185
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,256,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,512,0.1222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,1024,0.1325
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,2048,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,4096,0.1693
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,8192,0.2126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,16384,0.3008
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,32768,0.4749
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,65536,0.8240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,128,1,1,131072,1.5165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,2,0.1320
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,4,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,8,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,16,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,32,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,64,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,128,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,256,0.1326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,512,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,1024,0.1569
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,2048,0.1819
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,4096,0.2264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,8192,0.3137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,16384,0.4906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,32768,0.8390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,65536,1.5400
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,256,1,1,131072,2.9504
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,2,0.1542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,4,0.1545
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,8,0.1551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,16,0.1547
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,32,0.1551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,64,0.1534
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,128,0.1557
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,256,0.1603
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,512,0.1732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,1024,0.1963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,2048,0.2417
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,4096,0.3330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,8192,0.5111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,16384,0.8683
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,32768,1.5832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,512,1,1,65536,3.0160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,2,0.2016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,4,0.2015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,8,0.2018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,16,0.2025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,32,0.2022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,64,0.2033
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,128,0.2031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,256,0.2073
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,512,0.2327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,1024,0.2758
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,2048,0.3596
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,4096,0.5290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,8192,0.8607
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,16384,1.5252
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,4,1024,1,1,32768,2.8514
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,2,0.0872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,4,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,8,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,16,0.0883
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,32,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,64,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,128,0.0881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,256,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,512,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,1024,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,2048,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,4096,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,8192,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,16384,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,32768,0.1014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,65536,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1,1,1,131072,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,2,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,4,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,8,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,16,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,32,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,64,0.0887
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,128,0.0881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,256,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,512,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,1024,0.0909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,2048,0.0912
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,4096,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,8192,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,16384,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,32768,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,65536,0.1190
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,2,1,1,131072,0.1328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,2,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,4,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,8,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,16,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,32,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,64,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,128,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,256,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,512,0.0991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,1024,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,2048,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,4096,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,8192,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,16384,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,32768,0.1203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,65536,0.1359
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,4,1,1,131072,0.1571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,2,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,4,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,8,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,16,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,32,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,64,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,128,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,256,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,512,0.1036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,1024,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,2048,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,4096,0.1066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,8192,0.1090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,16384,0.1217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,32768,0.1369
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,65536,0.1574
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,8,1,1,131072,0.1990
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,2,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,4,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,8,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,16,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,32,0.1066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,64,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,128,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,256,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,512,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,1024,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,2048,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,4096,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,8192,0.1241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,16384,0.1368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,32768,0.1599
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,65536,0.2030
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,16,1,1,131072,0.2850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,2,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,4,0.1096
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,8,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,16,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,32,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,64,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,128,0.1092
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,256,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,512,0.1107
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,1024,0.1117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,2048,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,4096,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,8192,0.1391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,16384,0.1618
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,32768,0.2050
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,65536,0.2919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,32,1,1,131072,0.4678
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,2,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,4,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,8,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,16,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,32,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,64,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,128,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,256,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,512,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,1024,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,2048,0.1265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,4096,0.1426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,8192,0.1644
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,16384,0.2078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,32768,0.2962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,65536,0.4713
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,64,1,1,131072,0.8199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,2,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,4,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,8,0.1177
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,16,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,32,0.1174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,64,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,128,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,256,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,512,0.1208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,1024,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,2048,0.1452
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,4096,0.1671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,8192,0.2126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,16384,0.2991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,32768,0.4737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,65536,0.8207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,128,1,1,131072,1.5140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,2,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,4,0.1264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,8,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,16,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,32,0.1268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,64,0.1265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,128,0.1258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,256,0.1284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,512,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,1024,0.1509
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,2048,0.1750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,4096,0.2203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,8192,0.3087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,16384,0.4831
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,32768,0.8334
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,65536,1.5381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,256,1,1,131072,2.9418
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,2,0.1466
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,4,0.1467
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,8,0.1475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,16,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,32,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,64,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,128,0.1485
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,256,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,512,0.1649
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,1024,0.1878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,2048,0.2347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,4096,0.3248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,8192,0.5025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,16384,0.8589
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,32768,1.5732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,512,1,1,65536,3.0061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,2,0.1873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,4,0.1859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,8,0.1861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,16,0.1854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,32,0.1854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,64,0.1865
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,128,0.1859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,256,0.1916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,512,0.2153
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,1024,0.2573
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,2048,0.3418
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,4096,0.5076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,8192,0.8405
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,16384,1.5044
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,2,1024,1,1,32768,2.8316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,2,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,4,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,8,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,16,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,32,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,64,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,128,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,256,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,512,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,1024,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,2048,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,4096,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,8192,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,16384,0.0965
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,32768,0.1008
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,65536,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1,1,1,131072,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,2,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,4,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,8,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,16,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,32,0.0883
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,64,0.0871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,128,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,256,0.0871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,512,0.0885
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,1024,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,2048,0.0885
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,4096,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,8192,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,16384,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,32768,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,65536,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,2,1,1,131072,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,2,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,4,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,8,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,16,0.0912
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,32,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,64,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,128,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,256,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,512,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,1024,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,2048,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,4096,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,8192,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,16384,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,32768,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,65536,0.1271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,4,1,1,131072,0.1504
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,2,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,4,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,8,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,16,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,32,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,64,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,128,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,256,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,512,0.0986
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,1024,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,2048,0.1009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,4096,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,8192,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,16384,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,32768,0.1312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,65536,0.1515
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,8,1,1,131072,0.1953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,2,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,4,0.1045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,8,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,16,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,32,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,64,0.1045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,128,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,256,0.1046
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,512,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,1024,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,2048,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,4096,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,8192,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,16384,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,32768,0.1597
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,65536,0.2002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,16,1,1,131072,0.2840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,2,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,4,0.1069
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,8,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,16,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,32,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,64,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,128,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,256,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,512,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,1024,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,2048,0.1117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,4096,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,8192,0.1364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,16384,0.1591
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,32768,0.2020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,65536,0.2899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,32,1,1,131072,0.4654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,2,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,4,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,8,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,16,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,32,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,64,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,128,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,256,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,512,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,1024,0.1108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,2048,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,4096,0.1377
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,8192,0.1600
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,16384,0.2033
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,32768,0.2922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,65536,0.4668
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,64,1,1,131072,0.8163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,2,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,4,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,8,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,16,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,32,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,64,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,128,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,256,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,512,0.1180
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,1024,0.1272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,2048,0.1423
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,4096,0.1642
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,8192,0.2083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,16384,0.2971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,32768,0.4703
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,65536,0.8173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,128,1,1,131072,1.5098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,2,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,4,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,8,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,16,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,32,0.1222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,64,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,128,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,256,0.1251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,512,0.1350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,1024,0.1487
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,2048,0.1721
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,4096,0.2169
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,8192,0.3047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,16384,0.4799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,32768,0.8301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,65536,1.5324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,256,1,1,131072,2.9328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,2,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,4,0.1404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,8,0.1407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,16,0.1408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,32,0.1407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,64,0.1388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,128,0.1427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,256,0.1467
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,512,0.1593
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,1024,0.1837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,2048,0.2272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,4096,0.3189
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,8192,0.4979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,16384,0.8537
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,32768,1.5679
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,512,1,1,65536,3.0017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,2,0.1775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,4,0.1775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,8,0.1794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,16,0.1782
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,32,0.1782
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,64,0.1773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,128,0.1772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,256,0.1835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,512,0.2066
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,1024,0.2481
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,2048,0.3328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,4096,0.4973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,8192,0.8305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,16384,1.4939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,fp8_block,1,1024,1,1,32768,2.8214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,2,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,4,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,8,0.1151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,16,0.1159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,32,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,64,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,128,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,256,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,512,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,1024,0.1209
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,2048,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,4096,0.1275
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,8192,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,16384,0.1455
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,32768,0.1368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,65536,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1,1,1,131072,0.1561
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,2,0.1184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,4,0.1203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,8,0.1209
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,16,0.1200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,32,0.1215
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,64,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,128,0.1195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,256,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,512,0.1237
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,1024,0.1252
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,2048,0.1283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,4096,0.1355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,8192,0.1450
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,16384,0.1368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,32768,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,65536,0.1554
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,2,1,1,131072,0.1780
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,2,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,4,0.1248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,8,0.1233
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,16,0.1245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,32,0.1234
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,64,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,128,0.1236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,256,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,512,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,1024,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,2048,0.1374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,4096,0.1462
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,8192,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,16384,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,32768,0.1530
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,65536,0.1769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,4,1,1,131072,0.2237
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,2,0.1286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,4,0.1319
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,8,0.1321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,16,0.1299
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,32,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,64,0.1301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,128,0.1290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,256,0.1293
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,512,0.1327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,1024,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,2048,0.1496
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,4096,0.1384
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,8192,0.1451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,16384,0.1584
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,32768,0.1826
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,65536,0.2291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,8,1,1,131072,0.3265
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,2,0.1445
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,4,0.1443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,8,0.1450
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,16,0.1448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,32,0.1448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,64,0.1436
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,128,0.1443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,256,0.1448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,512,0.1491
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,1024,0.1585
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,2048,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,4096,0.1557
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,8192,0.1673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,16384,0.1954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,32768,0.2451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,65536,0.3492
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,16,1,1,131072,0.5523
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,2,0.1469
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,4,0.1469
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,8,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,16,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,32,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,64,0.1472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,128,0.1469
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,256,0.1487
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,512,0.1556
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,1024,0.1597
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,2048,0.1662
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,4096,0.1790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,8192,0.2044
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,16384,0.2566
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,32768,0.3601
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,65536,0.5650
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,32,1,1,131072,0.9745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,2,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,4,0.1629
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,8,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,16,0.1632
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,32,0.1654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,64,0.1635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,128,0.1657
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,256,0.1633
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,512,0.1704
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,1024,0.1750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,2048,0.1883
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,4096,0.2157
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,8192,0.2693
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,16384,0.3712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,32768,0.5744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,65536,0.9834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,64,1,1,131072,1.7990
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,2,0.2018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,4,0.2022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,8,0.2024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,16,0.2025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,32,0.2026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,64,0.2020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,128,0.2027
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,256,0.2060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,512,0.2122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,1024,0.2268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,2048,0.2560
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,4096,0.3077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,8192,0.4105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,16384,0.6143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,32768,1.0249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,65536,1.8439
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,128,1,1,131072,3.4845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,2,0.2743
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,4,0.2741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,8,0.2751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,16,0.2757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,32,0.2752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,64,0.2744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,128,0.2760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,256,0.2832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,512,0.2992
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,1024,0.3288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,2048,0.3810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,4096,0.4870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,8192,0.6918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,16384,1.1032
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,32768,1.9341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,65536,3.5864
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,256,1,1,131072,6.8948
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,2,0.4446
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,4,0.4447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,8,0.4441
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,16,0.4446
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,32,0.4457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,64,0.4442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,128,0.4465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,256,0.4602
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,512,0.4864
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,1024,0.5376
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,2048,0.6382
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,4096,0.8255
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,8192,1.2031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,16384,1.9528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,32768,3.4601
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,512,1,1,65536,6.4801
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,2,0.7639
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,4,0.7640
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,8,0.7631
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,16,0.7644
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,32,0.7626
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,64,0.7635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,128,0.7708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,256,0.7929
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,512,0.8440
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,1024,0.9416
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,2048,1.1414
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,4096,1.5157
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,8192,2.2700
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,16384,3.7766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,128,1024,1,1,32768,6.8007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,2,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,4,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,8,0.0886
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,16,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,32,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,64,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,128,0.0885
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,256,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,512,0.0906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,1024,0.0952
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,2048,0.0990
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,4096,0.0989
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,8192,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,16384,0.1136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,32768,0.1270
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,65536,0.1283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1,1,1,131072,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,2,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,4,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,8,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,16,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,32,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,64,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,128,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,256,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,512,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,1024,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,2048,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,4096,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,8192,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,16384,0.1296
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,32768,0.1347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,65536,0.1473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,2,1,1,131072,0.1744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,2,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,4,0.0972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,8,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,16,0.0971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,32,0.0967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,64,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,128,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,256,0.0973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,512,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,1024,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,2048,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,4096,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,8192,0.1325
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,16384,0.1343
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,32768,0.1474
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,65536,0.1721
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,4,1,1,131072,0.2216
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,2,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,4,0.1034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,8,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,16,0.1030
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,32,0.1026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,64,0.1034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,128,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,256,0.1029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,512,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,1024,0.1110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,2048,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,4096,0.1341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,8192,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,16384,0.1483
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,32768,0.1746
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,65536,0.2246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,8,1,1,131072,0.3256
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,2,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,4,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,8,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,16,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,32,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,64,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,128,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,256,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,512,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,1024,0.1306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,2048,0.1450
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,4096,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,8192,0.1538
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,16384,0.1795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,32768,0.2294
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,65536,0.3372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,16,1,1,131072,0.5387
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,2,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,4,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,8,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,16,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,32,0.1207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,64,0.1211
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,128,0.1206
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,256,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,512,0.1326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,1024,0.1453
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,2048,0.1451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,4096,0.1593
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,8192,0.1861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,16384,0.2379
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,32768,0.3434
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,65536,0.5468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,32,1,1,131072,0.9625
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,2,0.1281
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,4,0.1284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,8,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,16,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,32,0.1287
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,64,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,128,0.1300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,256,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,512,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,1024,0.1546
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,2048,0.1693
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,4096,0.1967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,8192,0.2503
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,16384,0.3515
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,32768,0.5582
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,65536,0.9769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,64,1,1,131072,1.7858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,2,0.1534
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,4,0.1536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,8,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,16,0.1535
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,32,0.1535
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,64,0.1525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,128,0.1561
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,256,0.1621
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,512,0.1683
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,1024,0.1836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,2048,0.2109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,4096,0.2653
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,8192,0.3639
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,16384,0.5659
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,32768,0.9885
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,65536,1.7972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,128,1,1,131072,3.5156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,2,0.1948
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,4,0.1955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,8,0.1943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,16,0.1944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,32,0.1954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,64,0.1946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,128,0.1960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,256,0.2048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,512,0.2204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,1024,0.2496
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,2048,0.3045
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,4096,0.4102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,8192,0.6171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,16384,1.0350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,32768,1.8927
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,65536,3.5406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,256,1,1,131072,6.9880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,2,0.2807
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,4,0.2794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,8,0.2791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,16,0.2801
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,32,0.2798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,64,0.2802
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,128,0.2848
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,256,0.3010
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,512,0.3309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,1024,0.3849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,2048,0.4971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,4096,0.7151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,8192,1.1415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,16384,1.9916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,32768,3.7317
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,512,1,1,65536,7.1293
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,2,0.4588
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,4,0.4587
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,8,0.4586
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,16,0.4603
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,32,0.4593
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,64,0.4580
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,128,0.4692
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,256,0.4917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,512,0.5436
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,1024,0.6491
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,2048,0.8559
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,4096,1.2576
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,8192,2.0627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,16384,3.6596
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,64,1024,1,1,32768,6.8810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,2,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,4,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,8,0.0822
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,16,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,32,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,64,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,128,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,256,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,512,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,1024,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,2048,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,4096,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,8192,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,16384,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,32768,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,65536,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1,1,1,131072,0.1508
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,2,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,4,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,8,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,16,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,32,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,64,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,128,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,256,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,512,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,1024,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,2048,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,4096,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,8192,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,16384,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,32768,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,65536,0.1408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,2,1,1,131072,0.1753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,2,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,4,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,8,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,16,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,32,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,64,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,128,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,256,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,512,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,1024,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,2048,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,4096,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,8192,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,16384,0.1277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,32768,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,65536,0.1791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,4,1,1,131072,0.2468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,2,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,4,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,8,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,16,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,32,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,64,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,128,0.0946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,256,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,512,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,1024,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,2048,0.1030
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,4096,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,8192,0.1301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,16384,0.1484
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,32768,0.1850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,65536,0.2531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,8,1,1,131072,0.3893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,2,0.1013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,4,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,8,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,16,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,32,0.1014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,64,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,128,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,256,0.1012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,512,0.1036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,1024,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,2048,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,4096,0.1334
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,8192,0.1536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,16384,0.1903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,32768,0.2676
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,65536,0.4170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,16,1,1,131072,0.7195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,2,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,4,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,8,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,16,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,32,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,64,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,128,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,256,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,512,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,1024,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,2048,0.1400
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,4096,0.1605
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,8192,0.1977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,16384,0.2725
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,32768,0.4254
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,65536,0.7258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,32,1,1,131072,1.3278
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,2,0.1146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,4,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,8,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,16,0.1152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,32,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,64,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,128,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,256,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,512,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,1024,0.1425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,2048,0.1611
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,4096,0.1991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,8192,0.2739
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,16384,0.4226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,32768,0.7219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,65536,1.3116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,64,1,1,131072,2.5093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,2,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,4,0.1325
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,8,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,16,0.1321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,32,0.1328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,64,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,128,0.1336
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,256,0.1391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,512,0.1536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,1024,0.1739
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,2048,0.2127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,4096,0.2886
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,8192,0.4399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,16384,0.7372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,32768,1.3427
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,65536,2.5678
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,128,1,1,131072,5.0232
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,2,0.1702
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,4,0.1685
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,8,0.1694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,16,0.1693
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,32,0.1673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,64,0.1686
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,128,0.1719
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,256,0.1745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,512,0.1967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,1024,0.2349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,2048,0.3103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,4096,0.4587
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,8192,0.7588
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,16384,1.3642
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,32768,2.6381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,65536,5.2508
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,256,1,1,131072,10.4588
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,2,0.2254
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,4,0.2254
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,8,0.2250
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,16,0.2255
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,32,0.2255
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,64,0.2246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,128,0.2277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,256,0.2326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,512,0.2712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,1024,0.3420
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,2048,0.4776
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,4096,0.7412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,8192,1.2741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,16384,2.4213
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,32768,4.8737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,512,1,1,65536,9.7143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,2,0.3411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,4,0.3424
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,8,0.3428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,16,0.3434
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,32,0.3435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,64,0.3426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,128,0.3488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,256,0.3572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,512,0.4301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,1024,0.5696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,2048,0.8426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,4096,1.3688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,8192,2.4373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,16384,4.7845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,32,1024,1,1,32768,9.7572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,2,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,4,0.0793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,8,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,16,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,32,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,64,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,128,0.0796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,256,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,512,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,1024,0.0840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,2048,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,4096,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,8192,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,16384,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,32768,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,65536,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1,1,1,131072,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,2,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,4,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,8,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,16,0.0819
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,32,0.0826
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,64,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,128,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,256,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,512,0.0829
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,1024,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,2048,0.0858
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,4096,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,8192,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,16384,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,32768,0.1111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,65536,0.1343
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,2,1,1,131072,0.1579
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,2,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,4,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,8,0.0845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,16,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,32,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,64,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,128,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,256,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,512,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,1024,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,2048,0.0881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,4096,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,8192,0.0945
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,16384,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,32768,0.1323
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,65536,0.1538
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,4,1,1,131072,0.2044
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,2,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,4,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,8,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,16,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,32,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,64,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,128,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,256,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,512,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,1024,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,2048,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,4096,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,8192,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,16384,0.1358
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,32768,0.1611
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,65536,0.2090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,8,1,1,131072,0.3229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,2,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,4,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,8,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,16,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,32,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,64,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,128,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,256,0.0985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,512,0.1008
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,1024,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,2048,0.1054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,4096,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,8192,0.1399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,16384,0.1669
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,32768,0.2156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,65536,0.3149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,16,1,1,131072,0.5110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,2,0.1028
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,4,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,8,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,16,0.1029
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,32,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,64,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,128,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,256,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,512,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,1024,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,2048,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,4096,0.1433
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,8192,0.1699
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,16384,0.2182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,32768,0.3233
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,65536,0.5251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,32,1,1,131072,0.9309
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,2,0.1055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,4,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,8,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,16,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,32,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,64,0.1049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,128,0.1054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,256,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,512,0.1086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,1024,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,2048,0.1473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,4096,0.1721
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,8192,0.2254
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,16384,0.3256
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,32768,0.5286
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,65536,0.9295
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,64,1,1,131072,1.7214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,2,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,4,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,8,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,16,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,32,0.1134
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,64,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,128,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,256,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,512,0.1301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,1024,0.1486
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,2048,0.1751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,4096,0.2243
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,8192,0.3275
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,16384,0.5296
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,32768,0.9262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,65536,1.7175
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,128,1,1,131072,3.3628
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,2,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,4,0.1292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,8,0.1313
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,16,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,32,0.1310
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,64,0.1306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,128,0.1346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,256,0.1428
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,512,0.1606
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,1024,0.1889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,2048,0.2400
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,4096,0.3414
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,8192,0.5421
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,16384,0.9468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,32768,1.7731
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,65536,3.3744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,256,1,1,131072,6.7258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,2,0.1712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,4,0.1719
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,8,0.1694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,16,0.1708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,32,0.1703
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,64,0.1714
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,128,0.1756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,256,0.1834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,512,0.2139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,1024,0.2656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,2048,0.3686
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,4096,0.5765
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,8192,0.9924
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,16384,1.8020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,32768,3.4637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,512,1,1,65536,6.7479
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,2,0.2390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,4,0.2389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,8,0.2386
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,16,0.2394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,32,0.2390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,64,0.2385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,128,0.2457
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,256,0.2578
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,512,0.3138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,1024,0.4080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,2048,0.6121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,4096,1.0001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,8192,1.7727
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,16384,3.3342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,16,1024,1,1,32768,6.4533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,2,0.0788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,4,0.0796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,8,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,16,0.0778
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,32,0.0777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,64,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,128,0.0788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,256,0.0784
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,512,0.0799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,1024,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,2048,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,4096,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,8192,0.0842
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,16384,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,32768,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,65536,0.1111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1,1,1,131072,0.1274
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,2,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,4,0.0788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,8,0.0793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,16,0.0797
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,32,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,64,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,128,0.0800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,256,0.0803
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,512,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,1024,0.0828
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,2048,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,4096,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,8192,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,16384,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,32768,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,65536,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,2,1,1,131072,0.1549
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,2,0.0800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,4,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,8,0.0809
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,16,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,32,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,64,0.0801
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,128,0.0797
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,256,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,512,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,1024,0.0829
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,2048,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,4096,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,8192,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,16384,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,32768,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,65536,0.1498
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,4,1,1,131072,0.2012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,2,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,4,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,8,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,16,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,32,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,64,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,128,0.0868
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,256,0.0871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,512,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,1024,0.0908
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,2048,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,4096,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,8192,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,16384,0.1289
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,32768,0.1526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,65536,0.2022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,8,1,1,131072,0.3034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,2,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,4,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,8,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,16,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,32,0.0952
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,64,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,128,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,256,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,512,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,1024,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,2048,0.1027
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,4096,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,8192,0.1344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,16384,0.1611
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,32768,0.2106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,65536,0.3103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,16,1,1,131072,0.5071
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,2,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,4,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,8,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,16,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,32,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,64,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,128,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,256,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,512,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,1024,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,2048,0.1129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,4096,0.1399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,8192,0.1626
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,16384,0.2126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,32768,0.3176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,65536,0.5186
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,32,1,1,131072,0.9200
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,2,0.1013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,4,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,8,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,16,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,32,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,64,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,128,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,256,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,512,0.1049
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,1024,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,2048,0.1389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,4096,0.1665
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,8192,0.2168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,16384,0.3167
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,32768,0.5212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,65536,0.9198
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,64,1,1,131072,1.7161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,2,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,4,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,8,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,16,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,32,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,64,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,128,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,256,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,512,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,1024,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,2048,0.1688
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,4096,0.2193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,8192,0.3191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,16384,0.5159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,32768,0.9196
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,65536,1.7025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,128,1,1,131072,3.3572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,2,0.1213
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,4,0.1215
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,8,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,16,0.1209
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,32,0.1208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,64,0.1201
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,128,0.1236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,256,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,512,0.1511
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,1024,0.1750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,2048,0.2272
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,4096,0.3308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,8192,0.5291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,16384,0.9303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,32768,1.7630
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,65536,3.3412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,256,1,1,131072,6.6909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,2,0.1433
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,4,0.1526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,8,0.1447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,16,0.1488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,32,0.1478
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,64,0.1484
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,128,0.1542
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,256,0.1628
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,512,0.1901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,1024,0.2415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,2048,0.3485
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,4096,0.5535
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,8192,0.9656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,16384,1.7725
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,32768,3.4465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,512,1,1,65536,6.7068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,2,0.2006
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,4,0.2001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,8,0.2013
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,16,0.2015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,32,0.2005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,64,0.2007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,128,0.2080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,256,0.2204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,512,0.2708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,1024,0.3663
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,2048,0.5679
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,4096,0.9595
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,8192,1.7312
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,16384,3.2847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,8,1024,1,1,32768,6.3805
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,2,0.0745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,4,0.0750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,8,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,16,0.0754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,32,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,64,0.0741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,128,0.0744
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,256,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,512,0.0769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,1024,0.0777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,2048,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,4096,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,8192,0.0800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,16384,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,32768,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,65536,0.1088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1,1,1,131072,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,2,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,4,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,8,0.0791
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,16,0.0793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,32,0.0796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,64,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,128,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,256,0.0793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,512,0.0810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,1024,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,2048,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,4096,0.0810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,8192,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,16384,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,32768,0.0985
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,65536,0.1333
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,2,1,1,131072,0.1557
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,2,0.0839
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,4,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,8,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,16,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,32,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,64,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,128,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,256,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,512,0.0857
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,1024,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,2048,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,4096,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,8192,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,16384,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,32768,0.1299
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,65536,0.1518
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,4,1,1,131072,0.2009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,2,0.0872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,4,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,8,0.0869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,16,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,32,0.0869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,64,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,128,0.0891
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,256,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,512,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,1024,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,2048,0.0930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,4096,0.0946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,8192,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,16384,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,32768,0.1531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,65536,0.2012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,8,1,1,131072,0.3157
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,2,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,4,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,8,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,16,0.0924
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,32,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,64,0.0923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,128,0.0924
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,256,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,512,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,1024,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,2048,0.0992
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,4096,0.1054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,8192,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,16384,0.1587
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,32768,0.2063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,65536,0.3031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,16,1,1,131072,0.4993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,2,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,4,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,8,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,16,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,32,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,64,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,128,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,256,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,512,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,1024,0.1012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,2048,0.1089
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,4096,0.1369
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,8192,0.1638
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,16384,0.2142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,32768,0.3165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,65536,0.5172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,32,1,1,131072,0.9225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,2,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,4,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,8,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,16,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,32,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,64,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,128,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,256,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,512,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,1024,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,2048,0.1393
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,4096,0.1645
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,8192,0.2170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,16384,0.3170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,32768,0.5156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,65536,0.9203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,64,1,1,131072,1.7117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,2,0.1054
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,4,0.1070
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,8,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,16,0.1048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,32,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,64,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,128,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,256,0.1055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,512,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,1024,0.1386
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,2048,0.1652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,4096,0.2166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,8192,0.3179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,16384,0.5194
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,32768,0.9162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,65536,1.6984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,128,1,1,131072,3.3507
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,2,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,4,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,8,0.1162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,16,0.1169
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,32,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,64,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,128,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,256,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,512,0.1442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,1024,0.1711
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,2048,0.2237
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,4096,0.3255
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,8192,0.5241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,16384,0.9260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,32768,1.7532
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,65536,3.3450
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,256,1,1,131072,6.6730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,2,0.1365
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,4,0.1404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,8,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,16,0.1360
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,32,0.1364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,64,0.1349
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,128,0.1447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,256,0.1544
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,512,0.1810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,1024,0.2342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,2048,0.3386
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,4096,0.5426
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,8192,0.9511
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,16384,1.7690
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,32768,3.4253
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,512,1,1,65536,6.6943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,2,0.1850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,4,0.1971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,8,0.1845
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,16,0.1854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,32,0.1851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,64,0.1856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,128,0.1916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,256,0.2041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,512,0.2547
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,1024,0.3477
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,2048,0.5484
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,4096,0.9318
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,8192,1.7021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,16384,3.2659
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,4,1024,1,1,32768,6.3552
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,2,0.0726
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,4,0.0721
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,8,0.0726
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,16,0.0729
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,32,0.0732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,64,0.0732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,128,0.0730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,256,0.0712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,512,0.0735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,1024,0.0756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,2048,0.0759
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,4096,0.0772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,8192,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,16384,0.0826
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,32768,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,65536,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1,1,1,131072,0.1249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,2,0.0748
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,4,0.0760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,8,0.0750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,16,0.0756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,32,0.0757
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,64,0.0756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,128,0.0753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,256,0.0751
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,512,0.0778
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,1024,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,2048,0.0792
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,4096,0.0795
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,8192,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,16384,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,32768,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,65536,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,2,1,1,131072,0.1525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,2,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,4,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,8,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,16,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,32,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,64,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,128,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,256,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,512,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,1024,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,2048,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,4096,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,8192,0.0903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,16384,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,32768,0.1283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,65536,0.1527
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,4,1,1,131072,0.2005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,2,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,4,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,8,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,16,0.0870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,32,0.0872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,64,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,128,0.0866
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,256,0.0866
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,512,0.0888
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,1024,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,2048,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,4096,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,8192,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,16384,0.1293
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,32768,0.1525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,65536,0.2026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,8,1,1,131072,0.3031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,2,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,4,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,8,0.0909
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,16,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,32,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,64,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,128,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,256,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,512,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,1024,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,2048,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,4096,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,8192,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,16384,0.1576
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,32768,0.2059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,65536,0.3059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,16,1,1,131072,0.5053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,2,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,4,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,8,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,16,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,32,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,64,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,128,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,256,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,512,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,1024,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,2048,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,4096,0.1355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,8192,0.1598
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,16384,0.2083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,32768,0.3143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,65536,0.5151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,32,1,1,131072,0.9187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,2,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,4,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,8,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,16,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,32,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,64,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,128,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,256,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,512,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,1024,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,2048,0.1381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,4096,0.1620
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,8192,0.2135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,16384,0.3159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,32768,0.5188
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,65536,0.9233
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,64,1,1,131072,1.7108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,2,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,4,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,8,0.1042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,16,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,32,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,64,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,128,0.1038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,256,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,512,0.1113
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,1024,0.1373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,2048,0.1638
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,4096,0.2144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,8192,0.3146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,16384,0.5162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,32768,0.9156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,65536,1.6966
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,128,1,1,131072,3.3397
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,2,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,4,0.1135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,8,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,16,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,32,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,64,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,128,0.1138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,256,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,512,0.1436
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,1024,0.1672
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,2048,0.2207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,4096,0.3207
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,8192,0.5193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,16384,0.9244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,32768,1.7521
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,65536,3.3406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,256,1,1,131072,6.6745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,2,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,4,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,8,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,16,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,32,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,64,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,128,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,256,0.1488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,512,0.1761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,1024,0.2263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,2048,0.3346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,4096,0.5396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,8192,0.9479
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,16384,1.7554
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,32768,3.4221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,512,1,1,65536,6.6676
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,2,0.1774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,4,0.1773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,8,0.1766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,16,0.1767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,32,0.1767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,64,0.1761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,128,0.1832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,256,0.1962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,512,0.2439
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,1024,0.3380
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,2048,0.5403
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,4096,0.9290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,8192,1.7020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,16384,3.2536
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,2,1024,1,1,32768,6.3526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,2,0.0712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,4,0.0716
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,8,0.0712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,16,0.0715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,32,0.0708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,64,0.0712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,128,0.0714
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,256,0.0712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,512,0.0730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,1024,0.0750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,2048,0.0752
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,4096,0.0756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,8192,0.0771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,16384,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,32768,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,65536,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1,1,1,131072,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,2,0.0740
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,4,0.0733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,8,0.0736
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,16,0.0737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,32,0.0733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,64,0.0745
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,128,0.0750
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,256,0.0766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,512,0.0767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,1024,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,2048,0.0777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,4096,0.0772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,8192,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,16384,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,32768,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,65536,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,2,1,1,131072,0.1499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,2,0.0773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,4,0.0777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,8,0.0773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,16,0.0768
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,32,0.0771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,64,0.0772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,128,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,256,0.0771
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,512,0.0788
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,1024,0.0799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,2048,0.0799
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,4096,0.0841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,8192,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,16384,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,32768,0.1226
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,65536,0.1478
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,4,1,1,131072,0.1948
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,2,0.0840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,4,0.0837
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,8,0.0841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,16,0.0836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,32,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,64,0.0844
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,128,0.0846
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,256,0.0843
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,512,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,1024,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,2048,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,4096,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,8192,0.0964
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,16384,0.1264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,32768,0.1493
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,65536,0.2004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,8,1,1,131072,0.3009
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,2,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,4,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,8,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,16,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,32,0.0903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,64,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,128,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,256,0.0906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,512,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,1024,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,2048,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,4096,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,8192,0.1299
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,16384,0.1572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,32768,0.2044
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,65536,0.3026
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,16,1,1,131072,0.5001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,2,0.0932
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,4,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,8,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,16,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,32,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,64,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,128,0.0931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,256,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,512,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,1024,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,2048,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,4096,0.1324
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,8192,0.1570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,16384,0.2058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,32768,0.3103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,65536,0.5102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,32,1,1,131072,0.9138
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,2,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,4,0.0963
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,8,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,16,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,32,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,64,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,128,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,256,0.0954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,512,0.0968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,1024,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,2048,0.1326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,4096,0.1591
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,8192,0.2093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,16384,0.3106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,32768,0.5129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,65536,0.9204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,64,1,1,131072,1.7096
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,2,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,4,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,8,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,16,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,32,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,64,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,128,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,256,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,512,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,1024,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,2048,0.1613
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,4096,0.2129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,8192,0.3118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,16384,0.5157
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,32768,0.9124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,65536,1.6915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,128,1,1,131072,3.3341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,2,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,4,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,8,0.1106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,16,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,32,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,64,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,128,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,256,0.1149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,512,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,1024,0.1643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,2048,0.2187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,4096,0.3195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,8192,0.5171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,16384,0.9205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,32768,1.7556
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,65536,3.3489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,256,1,1,131072,6.6713
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,2,0.1261
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,4,0.1263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,8,0.1264
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,16,0.1279
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,32,0.1276
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,64,0.1268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,128,0.1343
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,256,0.1451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,512,0.1724
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,1024,0.2236
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,2048,0.3298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,4096,0.5342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,8192,0.9448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,16384,1.7565
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,32768,3.4204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,512,1,1,65536,6.6741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,2,0.1693
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,4,0.1701
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,8,0.1699
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,16,0.1696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,32,0.1694
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,64,0.1695
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,128,0.1761
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,256,0.1889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,512,0.2371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,1024,0.3291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,2048,0.5322
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,4096,0.9222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,8192,1.6930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,16384,3.2425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,bfloat16,nvfp4,1,1024,1,1,32768,6.3501
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,2,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,4,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,8,0.1111
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,16,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,32,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,64,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,128,0.1109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,256,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,512,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,1024,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,2048,0.1151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,4096,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,8192,0.1213
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,16384,0.1274
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,32768,0.1316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,65536,0.1353
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1,1,1,131072,0.1420
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,2,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,4,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,8,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,16,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,32,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,64,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,128,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,256,0.1155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,512,0.1152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,1024,0.1182
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,2048,0.1193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,4096,0.1245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,8192,0.1277
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,16384,0.1311
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,32768,0.1334
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,65536,0.1399
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,2,1,1,131072,0.1512
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,2,0.1186
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,4,0.1176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,8,0.1185
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,16,0.1179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,32,0.1190
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,64,0.1188
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,128,0.1190
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,256,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,512,0.1193
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,1024,0.1233
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,2048,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,4096,0.1326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,8192,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,16384,0.1339
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,32768,0.1419
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,65536,0.1519
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,4,1,1,131072,0.1754
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,2,0.1258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,4,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,8,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,16,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,32,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,64,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,128,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,256,0.1225
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,512,0.1246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,1024,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,2048,0.1332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,4096,0.1332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,8192,0.1364
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,16384,0.1446
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,32768,0.1554
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,65536,0.1769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,8,1,1,131072,0.2245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,2,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,4,0.1337
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,8,0.1330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,16,0.1329
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,32,0.1341
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,64,0.1342
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,128,0.1332
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,256,0.1321
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,512,0.1347
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,1024,0.1368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,2048,0.1406
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,4096,0.1416
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,8192,0.1512
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,16384,0.1614
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,32768,0.1890
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,65536,0.2388
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,16,1,1,131072,0.3404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,2,0.1358
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,4,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,8,0.1362
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,16,0.1370
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,32,0.1360
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,64,0.1372
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,128,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,256,0.1371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,512,0.1432
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,1024,0.1465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,2048,0.1489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,4096,0.1565
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,8192,0.1697
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,16384,0.1944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,32768,0.2460
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,65536,0.3475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,32,1,1,131072,0.5505
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,2,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,4,0.1429
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,8,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,16,0.1435
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,32,0.1433
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,64,0.1434
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,128,0.1423
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,256,0.1446
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,512,0.1471
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,1024,0.1505
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,2048,0.1566
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,4096,0.1701
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,8192,0.1960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,16384,0.2469
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,32768,0.3466
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,65536,0.5558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,64,1,1,131072,0.9636
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,2,0.1640
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,4,0.1643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,8,0.1657
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,16,0.1638
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,32,0.1647
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,64,0.1641
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,128,0.1652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,256,0.1666
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,512,0.1698
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,1024,0.1774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,2048,0.1899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,4096,0.2181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,8192,0.2687
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,16384,0.3719
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,32768,0.5820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,65536,0.9946
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,128,1,1,131072,1.8103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,2,0.2038
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,4,0.2036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,8,0.2040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,16,0.2042
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,32,0.2041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,64,0.2031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,128,0.2036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,256,0.2060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,512,0.2140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,1024,0.2271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,2048,0.2551
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,4096,0.3110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,8192,0.4121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,16384,0.6240
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,32768,1.0456
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,65536,1.8655
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,256,1,1,131072,3.5090
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,2,0.2903
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,4,0.2924
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,8,0.2917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,16,0.2911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,32,0.2906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,64,0.2913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,128,0.2899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,256,0.2952
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,512,0.3088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,1024,0.3351
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,2048,0.3862
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,4096,0.4847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,8192,0.6715
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,16384,1.0421
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,32768,1.7831
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,512,1,1,65536,3.2623
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,2,0.4667
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,4,0.4664
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,8,0.4674
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,16,0.4671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,32,0.4669
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,64,0.4654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,128,0.4666
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,256,0.4717
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,512,0.4991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,1024,0.5504
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,2048,0.6494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,4096,0.8437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,8192,1.2168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,16384,1.9529
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,128,1024,1,1,32768,3.4355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,2,0.0862
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,4,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,8,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,16,0.0867
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,32,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,64,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,128,0.0894
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,256,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,512,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,1024,0.0925
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,2048,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,4096,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,8192,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,16384,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,32768,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,65536,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1,1,1,131072,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,2,0.0930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,4,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,8,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,16,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,32,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,64,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,128,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,256,0.0944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,512,0.0944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,1024,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,2048,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,4096,0.0975
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,8192,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,16384,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,32768,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,65536,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,2,1,1,131072,0.1422
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,2,0.0947
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,4,0.0954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,8,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,16,0.0948
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,32,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,64,0.0950
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,128,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,256,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,512,0.0989
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,1024,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,2048,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,4096,0.1025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,8192,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,16384,0.1241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,32768,0.1290
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,65536,0.1407
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,4,1,1,131072,0.1640
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,2,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,4,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,8,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,16,0.1036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,32,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,64,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,128,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,256,0.1025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,512,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,1024,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,2048,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,4096,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,8192,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,16384,0.1331
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,32768,0.1411
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,65536,0.1632
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,8,1,1,131072,0.2069
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,2,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,4,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,8,0.1092
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,16,0.1086
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,32,0.1087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,64,0.1091
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,128,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,256,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,512,0.1089
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,1024,0.1137
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,2048,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,4096,0.1291
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,8192,0.1355
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,16384,0.1451
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,32768,0.1692
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,65536,0.2123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,16,1,1,131072,0.3017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,2,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,4,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,8,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,16,0.1135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,32,0.1145
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,64,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,128,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,256,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,512,0.1195
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,1024,0.1245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,2048,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,4096,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,8192,0.1491
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,16384,0.1738
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,32768,0.2214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,65536,0.3159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,32,1,1,131072,0.5076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,2,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,4,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,8,0.1187
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,16,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,32,0.1208
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,64,0.1179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,128,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,256,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,512,0.1319
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,1024,0.1362
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,2048,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,4096,0.1552
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,8192,0.1776
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,16384,0.2269
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,32768,0.3211
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,65536,0.5159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,64,1,1,131072,0.8905
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,2,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,4,0.1362
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,8,0.1361
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,16,0.1346
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,32,0.1365
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,64,0.1365
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,128,0.1369
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,256,0.1387
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,512,0.1431
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,1024,0.1494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,2048,0.1626
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,4096,0.1860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,8192,0.2330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,16384,0.3283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,32768,0.5241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,65536,0.8997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,128,1,1,131072,1.6489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,2,0.1617
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,4,0.1613
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,8,0.1629
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,16,0.1611
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,32,0.1609
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,64,0.1621
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,128,0.1617
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,256,0.1653
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,512,0.1724
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,1024,0.1854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,2048,0.2122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,4096,0.2612
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,8192,0.3570
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,16384,0.5554
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,32768,0.9403
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,65536,1.6979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,256,1,1,131072,3.2152
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,2,0.2171
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,4,0.2184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,8,0.2166
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,16,0.2174
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,32,0.2183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,64,0.2186
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,128,0.2170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,256,0.2238
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,512,0.2382
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,1024,0.2650
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,2048,0.3160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,4096,0.4160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,8192,0.6159
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,16384,1.0093
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,32768,1.7809
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,512,1,1,65536,3.3252
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,2,0.3249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,4,0.3268
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,8,0.3256
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,16,0.3257
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,32,0.3260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,64,0.3261
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,128,0.3267
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,256,0.3398
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,512,0.3635
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,1024,0.4143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,2048,0.5079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,4096,0.6880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,8192,1.0403
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,16384,1.7503
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,64,1024,1,1,32768,3.1601
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,2,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,4,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,8,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,16,0.0893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,32,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,64,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,128,0.0889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,256,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,512,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,1024,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,2048,0.0923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,4096,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,8192,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,16384,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,32768,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,65536,0.1116
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1,1,1,131072,0.1299
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,2,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,4,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,8,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,16,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,32,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,64,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,128,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,256,0.0914
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,512,0.0935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,1024,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,2048,0.0937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,4096,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,8192,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,16384,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,32768,0.1136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,65536,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,2,1,1,131072,0.1437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,2,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,4,0.0972
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,8,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,16,0.0971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,32,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,64,0.0979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,128,0.0977
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,256,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,512,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,1024,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,2048,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,4096,0.1035
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,8192,0.1056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,16384,0.1162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,32768,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,65536,0.1465
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,4,1,1,131072,0.1759
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,2,0.1025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,4,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,8,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,16,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,32,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,64,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,128,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,256,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,512,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,1024,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,2048,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,4096,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,8192,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,16384,0.1356
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,32768,0.1526
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,65536,0.1821
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,8,1,1,131072,0.2443
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,2,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,4,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,8,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,16,0.1087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,32,0.1087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,64,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,128,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,256,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,512,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,1024,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,2048,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,4096,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,8192,0.1367
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,16384,0.1558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,32768,0.1900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,65536,0.2592
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,16,1,1,131072,0.3936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,2,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,4,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,8,0.1161
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,16,0.1151
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,32,0.1155
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,64,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,128,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,256,0.1142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,512,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,1024,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,2048,0.1297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,4096,0.1455
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,8192,0.1636
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,16384,0.1970
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,32768,0.2654
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,65536,0.3999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,32,1,1,131072,0.6730
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,2,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,4,0.1224
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,8,0.1221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,16,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,32,0.1227
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,64,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,128,0.1219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,256,0.1228
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,512,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,1024,0.1371
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,2048,0.1489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,4096,0.1675
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,8192,0.2035
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,16384,0.2702
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,32768,0.4048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,65536,0.6767
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,64,1,1,131072,1.2164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,2,0.1494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,4,0.1482
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,8,0.1499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,16,0.1492
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,32,0.1488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,64,0.1476
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,128,0.1494
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,256,0.1525
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,512,0.1588
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,1024,0.1698
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,2048,0.1881
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,4096,0.2247
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,8192,0.2940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,16384,0.4292
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,32768,0.7036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,65536,1.2540
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,128,1,1,131072,2.3732
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,2,0.1921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,4,0.1920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,8,0.1920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,16,0.1927
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,32,0.1921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,64,0.1920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,128,0.1937
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,256,0.1938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,512,0.2037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,1024,0.2221
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,2048,0.2595
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,4096,0.3282
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,8192,0.4627
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,16384,0.7350
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,32768,1.2829
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,65536,2.4056
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,256,1,1,131072,4.6973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,2,0.2667
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,4,0.2673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,8,0.2667
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,16,0.2675
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,32,0.2676
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,64,0.2664
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,128,0.2656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,256,0.2656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,512,0.2838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,1024,0.3179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,2048,0.3815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,4096,0.5050
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,8192,0.7434
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,16384,1.2306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,32768,2.2271
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,512,1,1,65536,4.3333
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,2,0.4408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,4,0.4404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,8,0.4408
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,16,0.4389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,32,0.4396
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,64,0.4395
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,128,0.4415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,256,0.4387
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,512,0.4696
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,1024,0.5313
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,2048,0.6585
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,4096,0.9059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,8192,1.3737
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,16384,2.3442
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,32,1024,1,1,32768,4.3424
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,2,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,4,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,8,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,16,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,32,0.0871
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,64,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,128,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,256,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,512,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,1024,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,2048,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,4096,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,8192,0.0920
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,16384,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,32768,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,65536,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1,1,1,131072,0.1183
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,2,0.0901
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,4,0.0890
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,8,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,16,0.0895
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,32,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,64,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,128,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,256,0.0891
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,512,0.0906
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,1024,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,2048,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,4096,0.0924
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,8192,0.0954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,16384,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,32768,0.1034
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,65536,0.1173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,2,1,1,131072,0.1340
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,2,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,4,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,8,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,16,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,32,0.0912
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,64,0.0922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,128,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,256,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,512,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,1024,0.0943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,2048,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,4096,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,8192,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,16384,0.1035
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,32768,0.1158
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,65536,0.1328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,4,1,1,131072,0.1558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,2,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,4,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,8,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,16,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,32,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,64,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,128,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,256,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,512,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,1024,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,2048,0.1039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,4096,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,8192,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,16384,0.1223
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,32768,0.1374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,65536,0.1585
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,8,1,1,131072,0.2002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,2,0.1051
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,4,0.1055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,8,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,16,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,32,0.1046
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,64,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,128,0.1050
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,256,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,512,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,1024,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,2048,0.1095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,4096,0.1122
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,8192,0.1249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,16384,0.1405
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,32768,0.1647
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,65536,0.2043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,16,1,1,131072,0.2875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,2,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,4,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,8,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,16,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,32,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,64,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,128,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,256,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,512,0.1117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,1024,0.1110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,2048,0.1162
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,4096,0.1254
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,8192,0.1437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,16384,0.1652
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,32768,0.2109
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,65536,0.2973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,32,1,1,131072,0.4733
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,2,0.1131
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,4,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,8,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,16,0.1128
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,32,0.1135
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,64,0.1123
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,128,0.1127
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,256,0.1124
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,512,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,1024,0.1184
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,2048,0.1333
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,4096,0.1479
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,8192,0.1700
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,16384,0.2139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,32768,0.3023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,65536,0.4779
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,64,1,1,131072,0.8258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,2,0.1232
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,4,0.1248
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,8,0.1238
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,16,0.1251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,32,0.1245
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,64,0.1242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,128,0.1241
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,256,0.1247
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,512,0.1288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,1024,0.1437
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,2048,0.1560
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,4096,0.1784
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,8192,0.2216
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,16384,0.3098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,32768,0.4855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,65536,0.8306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,128,1,1,131072,1.5253
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,2,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,4,0.1473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,8,0.1472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,16,0.1467
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,32,0.1488
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,64,0.1469
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,128,0.1493
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,256,0.1528
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,512,0.1637
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,1024,0.1753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,2048,0.1998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,4096,0.2455
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,8192,0.3330
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,16384,0.5085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,32768,0.8584
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,65536,1.5610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,256,1,1,131072,2.9671
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,2,0.1930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,4,0.1933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,8,0.1936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,16,0.1922
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,32,0.1936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,64,0.1931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,128,0.1938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,256,0.1980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,512,0.2120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,1024,0.2373
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,2048,0.2824
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,4096,0.3741
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,8192,0.5531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,16384,0.9092
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,32768,1.6251
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,512,1,1,65536,3.0596
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,2,0.2766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,4,0.2774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,8,0.2769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,16,0.2773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,32,0.2777
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,64,0.2773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,128,0.2774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,256,0.2830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,512,0.3108
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,1024,0.3538
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,2048,0.4404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,4096,0.6099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,8192,0.9415
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,16384,1.6046
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,16,1024,1,1,32768,2.9344
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,2,0.0855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,4,0.0849
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,8,0.0855
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,16,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,32,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,64,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,128,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,256,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,512,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,1024,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,2048,0.0883
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,4096,0.0897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,8192,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,16384,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,32768,0.0984
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,65536,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1,1,1,131072,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,2,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,4,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,8,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,16,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,32,0.0872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,64,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,128,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,256,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,512,0.0885
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,1024,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,2048,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,4096,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,8192,0.0950
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,16384,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,32768,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,65536,0.1113
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,2,1,1,131072,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,2,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,4,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,8,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,16,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,32,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,64,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,128,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,256,0.0876
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,512,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,1024,0.0893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,2048,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,4096,0.0923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,8192,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,16384,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,32768,0.1068
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,65536,0.1263
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,4,1,1,131072,0.1479
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,2,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,4,0.0950
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,8,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,16,0.0954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,32,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,64,0.0959
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,128,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,256,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,512,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,1024,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,2048,0.0989
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,4096,0.1006
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,8192,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,16384,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,32768,0.1308
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,65536,0.1523
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,8,1,1,131072,0.1931
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,2,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,4,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,8,0.1031
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,16,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,32,0.1032
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,64,0.1024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,128,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,256,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,512,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,1024,0.1053
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,2048,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,4096,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,8192,0.1170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,16384,0.1366
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,32768,0.1585
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,65536,0.2004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,16,1,1,131072,0.2835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,2,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,4,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,8,0.1073
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,16,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,32,0.1074
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,64,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,128,0.1065
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,256,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,512,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,1024,0.1095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,2048,0.1126
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,4096,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,8192,0.1392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,16384,0.1604
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,32768,0.2039
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,65536,0.2924
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,32,1,1,131072,0.4711
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,2,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,4,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,8,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,16,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,32,0.1096
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,64,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,128,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,256,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,512,0.1098
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,1024,0.1129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,2048,0.1220
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,4096,0.1400
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,8192,0.1616
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,16384,0.2063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,32768,0.2935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,65536,0.4701
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,64,1,1,131072,0.8205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,2,0.1163
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,4,0.1160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,8,0.1144
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,16,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,32,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,64,0.1149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,128,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,256,0.1147
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,512,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,1024,0.1274
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,2048,0.1447
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,4096,0.1673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,8192,0.2113
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,16384,0.2991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,32768,0.4726
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,65536,0.8191
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,128,1,1,131072,1.5142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,2,0.1274
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,4,0.1283
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,8,0.1269
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,16,0.1275
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,32,0.1285
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,64,0.1288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,128,0.1287
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,256,0.1297
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,512,0.1430
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,1024,0.1559
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,2048,0.1784
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,4096,0.2242
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,8192,0.3117
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,16384,0.4886
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,32768,0.8391
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,65536,1.5417
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,256,1,1,131072,2.9441
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,2,0.1571
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,4,0.1572
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,8,0.1581
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,16,0.1569
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,32,0.1578
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,64,0.1562
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,128,0.1592
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,256,0.1628
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,512,0.1760
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,1024,0.2014
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,2048,0.2473
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,4096,0.3380
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,8192,0.5164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,16384,0.8735
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,32768,1.5886
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,512,1,1,65536,3.0244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,2,0.2146
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,4,0.2142
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,8,0.2136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,16,0.2136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,32,0.2129
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,64,0.2136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,128,0.2136
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,256,0.2205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,512,0.2463
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,1024,0.2892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,2048,0.3753
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,4096,0.5444
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,8192,0.8756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,16384,1.5412
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,8,1024,1,1,32768,2.8673
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,2,0.0807
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,4,0.0790
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,8,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,16,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,32,0.0816
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,64,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,128,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,256,0.0800
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,512,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,1024,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,2048,0.0832
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,4096,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,8192,0.0856
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,16384,0.0889
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,32768,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,65536,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1,1,1,131072,0.1102
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,2,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,4,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,8,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,16,0.0852
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,32,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,64,0.0861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,128,0.0859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,256,0.0862
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,512,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,1024,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,2048,0.0879
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,4096,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,8192,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,16384,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,32768,0.0993
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,65536,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,2,1,1,131072,0.1318
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,2,0.0902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,4,0.0896
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,8,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,16,0.0898
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,32,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,64,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,128,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,256,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,512,0.0921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,1024,0.0923
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,2048,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,4096,0.0952
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,8192,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,16384,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,32768,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,65536,0.1284
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,4,1,1,131072,0.1482
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,2,0.0941
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,4,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,8,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,16,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,32,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,64,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,128,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,256,0.0942
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,512,0.0953
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,1024,0.0957
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,2048,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,4096,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,8192,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,16384,0.1115
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,32768,0.1288
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,65536,0.1519
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,8,1,1,131072,0.1926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,2,0.1000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,4,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,8,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,16,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,32,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,64,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,128,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,256,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,512,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,1024,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,2048,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,4096,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,8192,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,16384,0.1335
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,32768,0.1561
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,65536,0.1968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,16,1,1,131072,0.2794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,2,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,4,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,8,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,16,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,32,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,64,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,128,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,256,0.1062
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,512,0.1075
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,1024,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,2048,0.1103
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,4096,0.1164
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,8192,0.1369
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,16384,0.1591
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,32768,0.2037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,65536,0.2902
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,32,1,1,131072,0.4670
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,2,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,4,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,8,0.1080
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,16,0.1085
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,32,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,64,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,128,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,256,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,512,0.1084
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,1024,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,2048,0.1194
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,4096,0.1390
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,8192,0.1610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,16384,0.2057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,32768,0.2926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,65536,0.4681
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,64,1,1,131072,0.8153
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,2,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,4,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,8,0.1106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,16,0.1105
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,32,0.1118
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,64,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,128,0.1112
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,256,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,512,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,1024,0.1219
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,2048,0.1392
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,4096,0.1609
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,8192,0.2060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,16384,0.2929
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,32768,0.4691
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,65536,0.8160
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,128,1,1,131072,1.5107
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,2,0.1212
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,4,0.1214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,8,0.1218
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,16,0.1214
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,32,0.1203
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,64,0.1204
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,128,0.1205
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,256,0.1202
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,512,0.1307
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,1024,0.1475
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,2048,0.1712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,4096,0.2156
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,8192,0.3047
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,16384,0.4783
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,32768,0.8300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,65536,1.5326
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,256,1,1,131072,2.9317
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,2,0.1363
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,4,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,8,0.1385
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,16,0.1386
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,32,0.1381
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,64,0.1389
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,128,0.1404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,256,0.1468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,512,0.1609
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,1024,0.1830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,2048,0.2293
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,4096,0.3199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,8192,0.4986
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,16384,0.8568
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,32768,1.5710
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,512,1,1,65536,3.0028
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,2,0.1865
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,4,0.1888
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,8,0.1859
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,16,0.1860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,32,0.1860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,64,0.1868
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,128,0.1863
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,256,0.1930
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,512,0.2179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,1024,0.2590
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,2048,0.3458
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,4096,0.5112
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,8192,0.8439
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,16384,1.5100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,4,1024,1,1,32768,2.8374
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,2,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,4,0.0796
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,8,0.0797
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,16,0.0793
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,32,0.0789
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,64,0.0772
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,128,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,256,0.0785
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,512,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,1024,0.0821
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,2048,0.0810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,4096,0.0818
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,8192,0.0828
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,16384,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,32768,0.0918
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,65536,0.0961
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1,1,1,131072,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,2,0.0817
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,4,0.0819
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,8,0.0810
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,16,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,32,0.0830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,64,0.0829
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,128,0.0828
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,256,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,512,0.0840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,1024,0.0838
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,2048,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,4096,0.0847
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,8192,0.0872
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,16384,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,32768,0.0956
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,65536,0.1041
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,2,1,1,131072,0.1287
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,2,0.0874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,4,0.0878
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,8,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,16,0.0873
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,32,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,64,0.0875
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,128,0.0877
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,256,0.0870
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,512,0.0892
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,1024,0.0899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,2048,0.0900
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,4096,0.0926
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,8192,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,16384,0.0967
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,32768,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,65536,0.1262
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,4,1,1,131072,0.1499
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,2,0.0936
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,4,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,8,0.0925
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,16,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,32,0.0940
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,64,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,128,0.0934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,256,0.0938
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,512,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,1024,0.0962
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,2048,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,4096,0.0998
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,8192,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,16384,0.1088
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,32768,0.1298
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,65536,0.1489
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,8,1,1,131072,0.1921
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,2,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,4,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,8,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,16,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,32,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,64,0.0983
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,128,0.0976
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,256,0.0974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,512,0.1001
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,1024,0.1005
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,2048,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,4096,0.1063
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,8192,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,16384,0.1327
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,32768,0.1535
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,65536,0.1944
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,16,1,1,131072,0.2773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,2,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,4,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,8,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,16,0.1015
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,32,0.1016
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,64,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,128,0.1023
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,256,0.1025
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,512,0.1037
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,1024,0.1052
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,2048,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,4096,0.1125
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,8192,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,16384,0.1558
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,32768,0.1991
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,65536,0.2874
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,32,1,1,131072,0.4618
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,2,0.1043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,4,0.1040
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,8,0.1058
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,16,0.1064
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,32,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,64,0.1057
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,128,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,256,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,512,0.1083
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,1024,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,2048,0.1172
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,4096,0.1368
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,8192,0.1593
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,16384,0.2043
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,32768,0.2913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,65536,0.4661
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,64,1,1,131072,0.8168
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,2,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,4,0.1100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,8,0.1097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,16,0.1106
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,32,0.1095
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,64,0.1097
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,128,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,256,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,512,0.1120
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,1024,0.1199
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,2048,0.1383
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,4096,0.1594
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,8192,0.2048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,16384,0.2919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,32768,0.4674
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,65536,0.8143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,128,1,1,131072,1.5077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,2,0.1170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,4,0.1176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,8,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,16,0.1175
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,32,0.1180
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,64,0.1179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,128,0.1179
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,256,0.1181
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,512,0.1266
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,1024,0.1425
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,2048,0.1656
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,4096,0.2107
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,8192,0.2989
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,16384,0.4743
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,32768,0.8231
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,65536,1.5261
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,256,1,1,131072,2.9273
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,2,0.1304
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,4,0.1303
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,8,0.1301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,16,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,32,0.1306
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,64,0.1301
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,128,0.1305
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,256,0.1394
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,512,0.1527
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,1024,0.1756
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,2048,0.2217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,4096,0.3110
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,8192,0.4897
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,16384,0.8468
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,32768,1.5631
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,512,1,1,65536,2.9935
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,2,0.1695
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,4,0.1698
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,8,0.1705
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,16,0.1699
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,32,0.1708
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,64,0.1707
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,128,0.1712
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,256,0.1770
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,512,0.2000
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,1024,0.2422
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,2048,0.3258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,4096,0.4934
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,8192,0.8258
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,16384,1.4891
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,2,1024,1,1,32768,2.8173
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,2,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,4,0.0773
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,8,0.0789
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,16,0.0769
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,32,0.0776
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,64,0.0775
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,128,0.0778
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,256,0.0774
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,512,0.0794
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,1024,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,2048,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,4096,0.0813
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,8192,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,16384,0.0860
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,32768,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,65536,0.0958
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1,1,1,131072,0.1067
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,2,0.0806
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,4,0.0798
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,8,0.0815
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,16,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,32,0.0812
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,64,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,128,0.0814
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,256,0.0809
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,512,0.0811
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,1024,0.0820
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,2048,0.0824
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,4096,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,8192,0.0869
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,16384,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,32768,0.0951
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,65536,0.1036
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,2,1,1,131072,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,2,0.0841
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,4,0.0850
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,8,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,16,0.0833
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,32,0.0834
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,64,0.0835
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,128,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,256,0.0830
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,512,0.0853
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,1024,0.0854
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,2048,0.0851
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,4096,0.0880
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,8192,0.0893
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,16384,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,32768,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,65536,0.1229
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,4,1,1,131072,0.1448
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,2,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,4,0.0911
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,8,0.0919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,16,0.0916
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,32,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,64,0.0915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,128,0.0913
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,256,0.0917
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,512,0.0933
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,1024,0.0939
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,2048,0.0955
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,4096,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,8192,0.0978
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,16384,0.1060
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,32768,0.1249
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,65536,0.1472
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,8,1,1,131072,0.1915
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,2,0.0968
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,4,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,8,0.0971
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,16,0.0981
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,32,0.0980
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,64,0.0982
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,128,0.0954
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,256,0.0960
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,512,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,1024,0.1012
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,2048,0.1003
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,4096,0.1032
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,8192,0.1101
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,16384,0.1302
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,32768,0.1531
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,65536,0.1943
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,16,1,1,131072,0.2766
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,2,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,4,0.0995
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,8,0.0994
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,16,0.0999
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,32,0.1004
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,64,0.1002
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,128,0.0997
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,256,0.0996
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,512,0.1007
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,1024,0.1021
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,2048,0.1050
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,4096,0.1104
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,8192,0.1300
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,16384,0.1520
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,32768,0.1974
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,65536,0.2840
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,32,1,1,131072,0.4592
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,2,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,4,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,8,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,16,0.1022
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,32,0.1020
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,64,0.1019
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,128,0.1018
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,256,0.1017
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,512,0.1032
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,1024,0.1048
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,2048,0.1121
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,4096,0.1316
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,8192,0.1533
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,16384,0.1979
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,32768,0.2861
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,65536,0.4622
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,64,1,1,131072,0.8114
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,2,0.1076
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,4,0.1081
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,8,0.1082
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,16,0.1078
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,32,0.1059
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,64,0.1061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,128,0.1079
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,256,0.1077
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,512,0.1099
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,1024,0.1165
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,2048,0.1362
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,4096,0.1578
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,8192,0.2024
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,16384,0.2899
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,32768,0.4643
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,65536,0.8100
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,128,1,1,131072,1.5061
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,2,0.1140
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,4,0.1150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,8,0.1139
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,16,0.1149
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,32,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,64,0.1148
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,128,0.1143
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,256,0.1141
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,512,0.1222
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,1024,0.1397
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,2048,0.1632
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,4096,0.2087
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,8192,0.2973
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,16384,0.4728
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,32768,0.8237
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,65536,1.5232
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,256,1,1,131072,2.9217
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,2,0.1259
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,4,0.1246
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,8,0.1243
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,16,0.1239
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,32,0.1244
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,64,0.1243
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,128,0.1260
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,256,0.1345
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,512,0.1470
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,1024,0.1703
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,2048,0.2150
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,4096,0.3055
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,8192,0.4848
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,16384,0.8404
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,32768,1.5563
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,512,1,1,65536,2.9884
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,2,0.1613
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,4,0.1632
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,8,0.1617
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,16,0.1610
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,32,0.1630
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,64,0.1616
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,128,0.1631
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,256,0.1679
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,512,0.1919
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,1024,0.2328
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,2048,0.3176
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,4096,0.4836
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,8192,0.8170
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,16384,1.4804
VLLM,0.19.0,NVIDIA GB200,mla_generation_module,default,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,bfloat16,fp8,nvfp4,1,1024,1,1,32768,2.8085
