framework,version,device,op_name,kernel_source,model,architecture,mla_dtype,kv_cache_dtype,gemm_type,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,1,1,0,0.0415
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,1,1,0,0.0731
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,4,1,0,0.0434
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,4,1,0,0.0743
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,8,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,8,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,16,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,16,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,32,1,0,0.0434
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,32,1,0,0.0741
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,64,1,0,0.0434
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,64,1,0,0.0743
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,1,1,0,0.0517
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,128,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,4,1,0,0.0537
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,256,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,8,1,0,0.0538
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,512,1,0,0.0475
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,16,1,0,0.0540
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,1024,1,0,0.0475
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,32,1,0,0.0537
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,2048,1,0,0.0477
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,1,1,0,0.1079
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,128,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,4,1,0,0.1120
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,4096,1,0,0.0497
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,256,1,0,0.0764
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,8,1,0,0.1123
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,512,1,0,0.0789
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,16,1,0,0.1115
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,1024,1,0,0.0804
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,32,1,0,0.1127
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,2048,1,0,0.0808
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,64,1,0,0.1122
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,4096,1,0,0.0807
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,8192,1,0,0.0508
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,128,1,0,0.1125
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,8192,1,0,0.0863
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,256,1,0,0.1144
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,512,1,0,0.1175
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,1024,1,0,0.1185
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,2048,1,0,0.1214
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1,16384,1,0,0.0560
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,4096,1,0,0.1275
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1,16384,1,0,0.0941
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,8192,1,0,0.1277
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,64,1,0,0.0541
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1,16384,1,0,0.1417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,128,1,0,0.0537
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,1,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,256,1,0,0.0559
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,1,1,0,0.1103
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,1,1,0,0.0716
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,4,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,512,1,0,0.0580
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,4,1,0,0.1096
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,4,1,0,0.0714
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,1024,1,0,0.0579
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,8,1,0,0.1095
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,8,1,0,0.0719
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,2048,1,0,0.0593
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,16,1,0,0.1102
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,16,1,0,0.0722
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,4096,1,0,0.0604
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,32,1,0,0.1101
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,32,1,0,0.0724
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,8192,1,0,0.0621
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,64,1,0,0.1101
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,64,1,0,0.0720
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,128,1,0,0.1103
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1,16384,1,0,0.0701
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,128,1,0,0.0722
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,256,1,0,0.1123
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,256,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,512,1,0,0.1159
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,512,1,0,0.0782
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,1024,1,0,0.1177
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,8,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,2048,1,0,0.1245
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,16,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,4096,1,0,0.1275
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,32,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,8192,1,0,0.1338
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,64,1,0,0.0457
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,2,16384,1,0,0.1270
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,128,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,256,1,0,0.0474
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,1,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,512,1,0,0.0478
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,4,1,0,0.0544
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,1024,1,0,0.0490
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,8,1,0,0.0541
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,1024,1,0,0.0785
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,2048,1,0,0.0477
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,16,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,1,1,0,0.1109
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,2048,1,0,0.0803
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,4096,1,0,0.0494
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,32,1,0,0.0556
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,4,1,0,0.1100
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,8192,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,64,1,0,0.0540
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,8,1,0,0.1099
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,128,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,2,16384,1,0,0.0579
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,16,1,0,0.1103
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,256,1,0,0.0559
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,1,1,0,0.0457
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,32,1,0,0.1099
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,512,1,0,0.0578
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,4,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,64,1,0,0.1095
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,1024,1,0,0.0579
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,8,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,128,1,0,0.1110
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,2048,1,0,0.0581
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,16,1,0,0.0457
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,256,1,0,0.1126
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,4096,1,0,0.0624
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,32,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,512,1,0,0.1167
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,8192,1,0,0.0661
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,64,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,4096,1,0,0.0850
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,1024,1,0,0.1235
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,2,16384,1,0,0.0796
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,128,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,8192,1,0,0.0959
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,2048,1,0,0.1271
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,1,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,256,1,0,0.0475
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,2,16384,1,0,0.1029
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,4096,1,0,0.1194
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,4,1,0,0.0546
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,512,1,0,0.0496
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,1,1,0,0.0733
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,8,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,8192,1,0,0.1220
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,1024,1,0,0.0496
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,4,1,0,0.0733
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,16,1,0,0.0556
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,2048,1,0,0.0496
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,4,16384,1,0,0.1285
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,8,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,32,1,0,0.0556
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,4096,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,16,1,0,0.0724
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,64,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,8192,1,0,0.0570
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,128,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,256,1,0,0.0571
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,4,16384,1,0,0.0641
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,512,1,0,0.0586
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,1,1,0,0.0828
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,1,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,1024,1,0,0.0597
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,4,1,0,0.0826
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,4,1,0,0.0453
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,2048,1,0,0.0600
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,8,1,0,0.0834
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,8,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,4096,1,0,0.0660
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,16,1,0,0.0832
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,16,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,8192,1,0,0.0761
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,32,1,0,0.0830
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,32,1,0,0.0447
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,64,1,0,0.0824
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,32,1,0,0.0728
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,128,1,0,0.0830
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,64,1,0,0.0741
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,256,1,0,0.0857
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,128,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,512,1,0,0.0948
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,256,1,0,0.0766
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,512,1,0,0.0804
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,1024,1,0,0.0989
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,1024,1,0,0.0805
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,2048,1,0,0.0887
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,2048,1,0,0.0903
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,4096,1,0,0.0914
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,4096,1,0,0.0946
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,4,16384,1,0,0.0892
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,8192,1,0,0.0993
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,8192,1,0,0.1011
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,1,1,0,0.0559
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,4,16384,1,0,0.0967
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,8,16384,1,0,0.1107
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,4,1,0,0.0560
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,8,1,0,0.0560
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,16,1,0,0.0559
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,32,1,0,0.0559
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,64,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,1,1,0,0.0887
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,128,1,0,0.0559
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,4,1,0,0.0879
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,64,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,256,1,0,0.0580
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,8,1,0,0.0861
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,128,1,0,0.0454
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,512,1,0,0.0602
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,16,1,0,0.0887
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,256,1,0,0.0475
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,1024,1,0,0.0618
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,32,1,0,0.0867
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,512,1,0,0.0477
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,1,1,0,0.0746
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,2048,1,0,0.0639
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,64,1,0,0.0872
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,1024,1,0,0.0496
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,4096,1,0,0.0760
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,128,1,0,0.0888
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,2048,1,0,0.0497
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,8192,1,0,0.0884
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,256,1,0,0.0916
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,4096,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,512,1,0,0.0947
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,8,16384,1,0,0.1085
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,8192,1,0,0.0631
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,1024,1,0,0.0903
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,1,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,2048,1,0,0.0941
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,8,16384,1,0,0.0876
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,4,1,0,0.0551
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,4096,1,0,0.1010
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,1,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,4,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,4,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,8192,1,0,0.1152
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,8,1,0,0.0753
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,8,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,16,1,0,0.0758
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,16,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,32,1,0,0.0752
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,32,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,16,16384,1,0,0.1394
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,64,1,0,0.0762
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,64,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,128,1,0,0.0754
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,128,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,256,1,0,0.0783
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,256,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,8,1,0,0.0554
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,512,1,0,0.0812
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,1,1,0,0.0825
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,16,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,1024,1,0,0.0914
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,4,1,0,0.0836
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,32,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,2048,1,0,0.0967
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,8,1,0,0.0825
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,4096,1,0,0.0889
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,16,1,0,0.0830
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,8192,1,0,0.0988
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,32,1,0,0.0833
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,64,1,0,0.0829
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,8,16384,1,0,0.1124
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,128,1,0,0.0845
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,512,1,0,0.0498
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,1,1,0,0.0599
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,256,1,0,0.0905
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,1024,1,0,0.0495
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,4,1,0,0.0595
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,2048,1,0,0.0517
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,512,1,0,0.0927
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,64,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,8,1,0,0.0578
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,4096,1,0,0.0617
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,128,1,0,0.0558
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,1024,1,0,0.0968
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,16,1,0,0.0599
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,256,1,0,0.0577
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,32,1,0,0.0581
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,8192,1,0,0.0864
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,2048,1,0,0.1032
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,512,1,0,0.0601
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,64,1,0,0.0588
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,1024,1,0,0.0621
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,128,1,0,0.0598
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,32,4096,1,0,0.1174
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,16,16384,1,0,0.1136
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,2048,1,0,0.0743
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,4096,1,0,0.0889
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,1,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,4,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,8192,1,0,0.1103
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,8,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,1,1,0,0.0898
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,16,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,4,1,0,0.0892
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,32,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,8,1,0,0.0892
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,64,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,256,1,0,0.0621
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,128,1,0,0.0458
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,512,1,0,0.0743
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,256,1,0,0.0476
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,1024,1,0,0.0808
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,512,1,0,0.0497
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,2048,1,0,0.0764
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,1024,1,0,0.0517
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,4096,1,0,0.0828
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,16,16384,1,0,0.1483
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,2048,1,0,0.0621
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,8192,1,0,0.0983
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,1,1,0,0.0475
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,16,1,0,0.0894
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,4,1,0,0.0476
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,32,1,0,0.0894
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,8,1,0,0.0475
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,16,16384,1,0,0.1233
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,64,1,0,0.0898
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,16,1,0,0.0475
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,1,1,0,0.0668
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,32,1,0,0.0474
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,128,1,0,0.0917
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,4,1,0,0.0680
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,64,1,0,0.0471
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,256,1,0,0.0925
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,8,1,0,0.0678
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,128,1,0,0.0479
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,512,1,0,0.0967
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,256,1,0,0.0520
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,32,4096,1,0,0.0860
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,512,1,0,0.0598
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,1024,1,0,0.1031
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,1,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,1024,1,0,0.0700
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,4,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,2048,1,0,0.0855
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,8,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,32,4096,1,0,0.1038
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,16,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,32,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,16,1,0,0.0681
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,1,1,0,0.0506
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,32,1,0,0.0684
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,4,1,0,0.0496
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,64,1,0,0.0437
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,64,1,0,0.0679
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,8,1,0,0.0508
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,128,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,128,1,0,0.0700
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,16,1,0,0.0497
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,2048,1,0,0.1171
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,256,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,256,1,0,0.0742
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,32,1,0,0.0505
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,512,1,0,0.0772
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,512,1,0,0.0517
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,64,1,0,0.0517
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,1024,1,0,0.0768
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,64,4096,1,0,0.1432
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,128,1,0,0.0538
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,1024,1,0,0.0660
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,2048,1,0,0.0851
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,256,1,0,0.0598
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,2048,1,0,0.0859
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,512,1,0,0.0686
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,1024,1,0,0.0824
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,1,1,0,0.1046
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,64,4096,1,0,0.1134
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,2048,1,0,0.1008
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,1,1,0,0.0437
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,4,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,64,4096,1,0,0.1383
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,8,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,32,4096,1,0,0.0988
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,1,1,0,0.0601
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,16,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,1,1,0,0.0641
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,4,1,0,0.0600
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,32,1,0,0.0436
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,4,1,0,0.0658
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,4,1,0,0.1049
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,8,1,0,0.0642
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,64,1,0,0.0437
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,16,1,0,0.0651
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,8,1,0,0.1052
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,128,1,0,0.0445
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,32,1,0,0.0658
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,16,1,0,0.1048
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,256,1,0,0.0498
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,64,1,0,0.0653
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,32,1,0,0.1048
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,128,1,0,0.0680
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,512,1,0,0.0661
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,64,1,0,0.1049
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,256,1,0,0.0762
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,8,1,0,0.0599
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,128,1,0,0.1073
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,16,1,0,0.0599
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,256,1,0,0.1091
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,32,1,0,0.0595
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,512,1,0,0.1169
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,64,1,0,0.0606
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,128,1,0,0.0626
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,1024,1,0,0.1305
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,256,1,0,0.0766
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,1024,1,0,0.0837
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,512,1,0,0.0891
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,128,2048,1,0,0.1596
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,128,2048,1,0,0.1104
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,1024,1,0,0.1077
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,128,2048,1,0,0.1473
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,512,1,0,0.0798
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,1,1,0,0.1312
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,1,1,0,0.0528
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,1024,1,0,0.0873
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,1,1,0,0.0802
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,4,1,0,0.1315
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,4,1,0,0.0530
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,2048,1,0,0.1028
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,8,1,0,0.1313
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,8,1,0,0.0541
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,16,1,0,0.1313
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,64,4096,1,0,0.1297
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,16,1,0,0.0531
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,32,1,0,0.1317
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,1,1,0,0.0722
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,32,1,0,0.0534
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,4,1,0,0.0720
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,64,1,0,0.1323
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,4,1,0,0.0801
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,8,1,0,0.0715
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,16,1,0,0.0720
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,8,1,0,0.0805
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,32,1,0,0.0722
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,16,1,0,0.0796
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,64,1,0,0.0724
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,32,1,0,0.0806
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,128,1,0,0.0758
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,64,1,0,0.0856
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,64,1,0,0.0564
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,256,1,0,0.0785
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,128,1,0,0.1372
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,128,1,0,0.0860
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,128,1,0,0.0622
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,512,1,0,0.0866
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,256,1,0,0.1454
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,256,1,0,0.1034
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,256,1,0,0.0789
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,1024,1,0,0.1023
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,512,1,0,0.1249
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,512,1,0,0.0943
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,128,2048,1,0,0.1291
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,256,1024,1,0,0.1635
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,1,1,0,0.0905
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,256,1024,1,0,0.1202
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,4,1,0,0.0908
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,8,1,0,0.0906
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,512,1,0,0.1612
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,16,1,0,0.0908
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,1,1,0,0.1175
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,1,1,0,0.0835
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,32,1,0,0.0910
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,256,1024,1,0,0.1897
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,64,1,0,0.0927
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,4,1,0,0.1170
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,4,1,0,0.0842
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,1,1,0,0.1880
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,128,1,0,0.0973
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,8,1,0,0.1166
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,8,1,0,0.0838
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,256,1,0,0.1055
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,4,1,0,0.1889
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,16,1,0,0.1177
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,16,1,0,0.0839
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,512,1,0,0.1224
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,8,1,0,0.1879
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,32,1,0,0.1170
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,32,1,0,0.0841
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,16,1,0,0.1881
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,64,1,0,0.1179
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,64,1,0,0.0894
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,32,1,0,0.1879
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,128,1,0,0.1201
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,128,1,0,0.0927
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,64,1,0,0.1911
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,256,1024,1,0,0.1506
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,256,1,0,0.1503
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,256,1,0,0.1150
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,128,1,0,0.2017
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,1,1,0,0.1263
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,512,512,1,0,0.1876
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,256,1,0,0.2163
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,4,1,0,0.1263
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,8,1,0,0.1259
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,512,512,1,0,0.2429
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,16,1,0,0.1261
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,512,512,1,0,0.1425
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,32,1,0,0.1261
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,1,1,0,0.3319
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,64,1,0,0.1324
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,1,1,0,0.1196
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,128,1,0,0.1430
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,4,1,0,0.3303
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,1,1,0,0.1771
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,4,1,0,0.1192
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,256,1,0,0.1586
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,8,1,0,0.3306
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,4,1,0,0.1767
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,8,1,0,0.1190
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,512,512,1,0,0.1881
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,16,1,0,0.3319
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,8,1,0,0.1765
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,16,1,0,0.1193
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,1,1,0,0.1963
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,32,1,0,0.3303
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,16,1,0,0.1772
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,32,1,0,0.1191
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,4,1,0,0.1953
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,32,1,0,0.1771
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,64,1,0,0.3326
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,64,1,0,0.1274
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,8,1,0,0.1958
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,64,1,0,0.1831
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,128,1,0,0.3497
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,16,1,0,0.1956
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,128,1,0,0.1328
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,128,1,0,0.1880
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,32,1,0,0.1956
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,128,1024,256,1,0,0.3823
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,16,1024,256,1,0,0.1703
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,64,1,0,0.2052
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,32,1024,256,1,0,0.2448
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,128,1,0,0.2234
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,64,1024,256,1,0,0.2503
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,1,1,0,0.0376
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,4,1,0,0.0394
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,8,1,0,0.0396
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,16,1,0,0.0394
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,32,1,0,0.0394
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,64,1,0,0.0396
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,128,1,0,0.0395
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,256,1,0,0.0415
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,512,1,0,0.0437
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,1024,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,2048,1,0,0.0437
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,4096,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,8192,1,0,0.0457
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1,16384,1,0,0.0503
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,1,1,0,0.0395
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,4,1,0,0.0397
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,8,1,0,0.0408
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,16,1,0,0.0397
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,32,1,0,0.0397
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,64,1,0,0.0396
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,128,1,0,0.0397
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,256,1,0,0.0416
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,512,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,1024,1,0,0.0433
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,2048,1,0,0.0434
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,4096,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,8192,1,0,0.0480
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,2,16384,1,0,0.0520
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,1,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,4,1,0,0.0415
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,8,1,0,0.0415
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,16,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,32,1,0,0.0415
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,64,1,0,0.0416
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,128,1,0,0.0414
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,256,1,0,0.0433
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,512,1,0,0.0439
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,1024,1,0,0.0438
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,2048,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,4096,1,0,0.0498
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,8192,1,0,0.0518
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,4,16384,1,0,0.0597
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,1,1,0,0.0396
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,4,1,0,0.0398
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,8,1,0,0.0397
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,16,1,0,0.0395
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,32,1,0,0.0396
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,64,1,0,0.0395
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,128,1,0,0.0396
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,256,1,0,0.0415
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,512,1,0,0.0435
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,1024,1,0,0.0437
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,2048,1,0,0.0455
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,4096,1,0,0.0498
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,8192,1,0,0.0557
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,8,16384,1,0,0.0823
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,1,1,0,0.0414
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,4,1,0,0.0412
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,8,1,0,0.0417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,16,1,0,0.0419
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,32,1,0,0.0413
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,64,1,0,0.0416
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,128,1,0,0.0415
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,256,1,0,0.0431
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,512,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,1024,1,0,0.0454
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,2048,1,0,0.0477
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,4096,1,0,0.0547
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,8192,1,0,0.0807
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,16,16384,1,0,0.1079
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,1,1,0,0.0417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,4,1,0,0.0417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,8,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,16,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,32,1,0,0.0428
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,64,1,0,0.0416
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,128,1,0,0.0421
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,256,1,0,0.0449
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,512,1,0,0.0477
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,1024,1,0,0.0478
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,2048,1,0,0.0562
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,32,4096,1,0,0.0806
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,1,1,0,0.0417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,4,1,0,0.0422
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,8,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,16,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,32,1,0,0.0430
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,64,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,128,1,0,0.0432
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,256,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,512,1,0,0.0497
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,1024,1,0,0.0572
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,2048,1,0,0.0806
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,64,4096,1,0,0.1093
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,1,1,0,0.0417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,4,1,0,0.0417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,8,1,0,0.0416
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,16,1,0,0.0416
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,32,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,64,1,0,0.0417
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,128,1,0,0.0418
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,256,1,0,0.0456
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,512,1,0,0.0564
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,1024,1,0,0.0798
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,128,2048,1,0,0.1078
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,1,1,0,0.0491
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,4,1,0,0.0496
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,8,1,0,0.0497
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,16,1,0,0.0488
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,32,1,0,0.0499
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,64,1,0,0.0498
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,128,1,0,0.0509
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,256,1,0,0.0704
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,512,1,0,0.0885
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,256,1024,1,0,0.1145
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,1,1,0,0.0681
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,4,1,0,0.0676
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,8,1,0,0.0676
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,16,1,0,0.0674
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,32,1,0,0.0684
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,64,1,0,0.0746
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,128,1,0,0.0832
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,256,1,0,0.1042
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,512,512,1,0,0.1317
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,1,1,0,0.1030
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,4,1,0,0.1028
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,8,1,0,0.1025
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,16,1,0,0.1030
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,32,1,0,0.1025
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,64,1,0,0.1110
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,128,1,0,0.1159
SGLang,0.5.10,NVIDIA GB200,mla_generation,trtllm_mla,deepseek-ai/DeepSeek-V3,DeepseekV3ForCausalLM,fp8_block,fp8,fp8_block,8,1024,256,1,0,0.1542
