framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,1,0.1746000051498413
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,3,0.1747648000717163
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,7,0.17548799514770508
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,15,0.1756816029548645
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,31,0.17624160051345825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,63,0.2052095890045166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,127,0.38335840702056884
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,255,0.750051212310791
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,511,1.5227328300476075
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,1023,3.1248783111572265
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,1,0.012174399942159653
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,3,0.012201599776744843
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,7,0.01215839982032776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,15,0.012150400131940842
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,31,0.012249600142240524
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,127,0.012681600451469422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,255,0.016657599806785585
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,511,0.02677760124206543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,1023,0.034334400296211244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,2047,0.05453919768333435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,1,0.012385600060224534
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,3,0.012398400157690049
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,15,0.01244639977812767
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,31,0.012460800260305405
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,63,0.01273919939994812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,127,0.013105599582195282
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,255,0.01723040044307709
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,511,0.03311359882354736
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,2047,6.434193420410156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,1023,0.05321440100669861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,1,0.09367679953575134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,3,0.09367200136184692
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,2047,0.2353663921356201
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,7,0.09376320242881775
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,63,0.01225920021533966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,31,0.0940064013004303
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,15,0.09343360066413879
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,63,0.09553120136260987
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,127,0.1955024003982544
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,255,0.4214159965515137
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,7,0.012383999675512314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,1,0.018092800676822663
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,3,0.018012799322605133
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,511,0.764851188659668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,7,0.01804800033569336
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,31,0.01815200001001358
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,63,0.01849599927663803
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,15,0.018225599825382233
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,127,0.0194255992770195
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,255,0.027577599883079527
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,511,0.05675680041313171
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,1023,1.5345392227172852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,1023,0.22625439167022704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,2047,0.4158336162567139
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,3,0.34266879558563235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,1,0.34300639629364016
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,15,0.34340479373931887
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,7,0.3420383930206299
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,31,0.3446671962738037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,63,0.39774560928344727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,127,0.7563248157501221
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,2047,3.280009460449219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,255,1.486625576019287
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,1,0.6768576145172119
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,3,0.6783840179443359
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,7,0.6762959957122803
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,15,0.6794159889221192
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,511,2.965595245361328
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,31,0.7070528030395508
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,63,0.7807295799255372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,127,1.63712158203125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,64,128,1,bfloat16,bfloat16,1,1.4009087562561036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,64,128,1,bfloat16,bfloat16,255,2.9669376373291017
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,64,128,1,bfloat16,bfloat16,3,1.349945640563965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,1023,6.070678329467773
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,64,128,1,bfloat16,bfloat16,7,1.3722592353820802
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,64,128,1,bfloat16,bfloat16,15,1.3763775825500488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,1,0.030811199545860292
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,3,0.03017120063304901
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,64,128,1,bfloat16,bfloat16,63,1.677459144592285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,7,0.030248001217842102
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,15,0.029713600873947144
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,31,0.029926401376724244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,63,0.03133440017700195
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,127,0.03171679973602295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,64,128,1,bfloat16,bfloat16,127,2.9981311798095702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,255,0.04870400130748749
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,511,0.21310720443725586
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,64,128,1,bfloat16,bfloat16,31,1.4464192390441895
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,1023,0.44155359268188477
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,2047,0.8706720352172852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,64,128,1,bfloat16,bfloat16,1,2.7463375091552735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,64,128,1,bfloat16,bfloat16,3,2.759675216674805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,64,128,1,bfloat16,bfloat16,15,2.8128719329833984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,64,128,1,bfloat16,bfloat16,31,2.87573127746582
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,64,128,1,bfloat16,bfloat16,7,2.768951988220215
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,64,128,1,bfloat16,bfloat16,1,5.6611183166503904
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,1,0.05403680205345154
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,64,128,1,bfloat16,bfloat16,3,5.7101184844970705
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,7,0.05403199791908264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,15,0.055011200904846194
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,64,128,1,bfloat16,bfloat16,63,3.0810400009155274
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,63,0.06178240180015564
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,64,128,1,bfloat16,bfloat16,7,5.676657485961914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,127,0.059299200773239136
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,511,0.40258078575134276
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,64,128,1,bfloat16,bfloat16,15,5.681790542602539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,1023,0.8681568145751953
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,3,0.05261279940605164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,1,0.13707200288772584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,3,0.14041919708251954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,7,0.13486239910125733
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,2047,1.6014144897460938
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,15,0.1370959997177124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,31,0.13688960075378417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,63,0.14649280309677123
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,31,0.0551584005355835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,255,0.19627039432525634
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,127,0.32177119255065917
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,255,0.5775055885314941
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,1,0.011825600266456604
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,3,0.011841599643230439
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,15,0.011814399808645248
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,7,0.011852800101041793
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,63,0.011878400295972823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,31,0.011879999935626984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,64,128,1,bfloat16,bfloat16,31,5.948723220825196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,127,0.012308800220489502
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,255,0.016169600188732147
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,511,1.134708786010742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,511,0.028110399842262268
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,1,0.011939200013875962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,2047,0.06329920291900634
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,15,0.011988800019025803
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,7,0.012068799883127212
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,31,0.012118399888277055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,63,0.01218079999089241
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,127,0.012824000418186187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,255,0.016540800034999848
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,3,0.012089599668979645
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,1023,0.05600960254669189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,511,0.03696959912776947
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,2047,0.13986879587173462
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,3,0.07055040001869202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,7,0.07039039731025695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,1,0.07075520157814026
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,1023,0.04161919951438904
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,31,0.07095680236816407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,15,0.07048320174217224
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,63,0.07200800180435181
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,127,0.08962720036506652
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,1,0.017752000689506532
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,3,0.017708800733089447
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,1023,2.294270324707031
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,7,0.017697599530220032
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,255,0.3192895889282227
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,31,0.017843200266361235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,511,0.5822703838348389
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,127,0.018828800320625304
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,15,0.017716799676418305
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,63,0.01828320026397705
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,511,0.05492159724235535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,255,0.026903998851776124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,1023,0.1486400008201599
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,1,0.2635567903518677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,1023,1.1566816329956056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,3,0.26336960792541503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,2047,0.3392672061920166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,15,0.26427199840545657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,7,0.26247999668121336
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,63,0.30588641166687014
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,31,0.2660720109939575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,2047,2.3454992294311525
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,3,0.5171887874603271
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,127,0.618995189666748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,2047,4.679457473754883
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,255,1.1236175537109374
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,1,0.5144544124603272
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,15,0.5183279991149903
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,31,0.5355535984039307
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,7,0.513481616973877
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,63,0.6454944133758544
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,127,1.137337589263916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,48,48,128,1,bfloat16,bfloat16,1,1.0247488021850586
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,511,2.240577507019043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,48,48,128,1,bfloat16,bfloat16,3,1.0194623947143555
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,48,48,128,1,bfloat16,bfloat16,7,1.015662384033203
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,1,0.02367199957370758
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,3,0.02354239970445633
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,7,0.023740799725055696
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,15,0.023606400191783904
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,48,48,128,1,bfloat16,bfloat16,15,1.0307663917541503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,31,0.023792000114917757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,63,0.024113599956035615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,255,0.037508800625801086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,127,0.025860801339149475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,511,0.13161120414733887
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,48,48,128,1,bfloat16,bfloat16,255,2.235655975341797
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,48,48,128,1,bfloat16,bfloat16,63,1.1919808387756348
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,48,48,128,1,bfloat16,bfloat16,31,1.0614048004150392
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,1023,0.311899209022522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,2047,0.6573071956634522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,48,48,128,1,bfloat16,bfloat16,127,2.265715217590332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,1023,4.558033752441406
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,48,48,128,1,bfloat16,bfloat16,1,2.010974311828613
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,48,48,128,1,bfloat16,bfloat16,3,2.0176191329956055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,48,48,128,1,bfloat16,bfloat16,7,2.11221923828125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,48,48,128,1,bfloat16,bfloat16,15,2.1071680068969725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,48,48,128,1,bfloat16,bfloat16,31,2.160539245605469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,1,0.04426400065422058
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,3,0.044203200936317445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,7,0.04421919882297516
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,48,48,128,1,bfloat16,bfloat16,63,2.4315887451171876
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,15,0.04415040016174317
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,63,0.04508480131626129
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,127,0.04675520062446594
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,255,0.07709599733352661
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,511,0.3063472032546997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,48,48,128,1,bfloat16,bfloat16,1,4.235982513427734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,48,48,128,1,bfloat16,bfloat16,3,4.254230499267578
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,1023,0.5944496154785156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,48,48,128,1,bfloat16,bfloat16,7,4.291960144042969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,1,0.11870239973068238
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,15,0.1189568042755127
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,48,48,128,1,bfloat16,bfloat16,15,4.295267105102539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,31,0.11930240392684936
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,7,0.11914080381393433
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,127,0.24358720779418946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,48,48,128,1,bfloat16,bfloat16,31,4.420187377929688
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,63,0.1234928011894226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,255,0.4760128021240234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,31,0.04674879908561706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,1,0.01178240031003952
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,7,0.011767999827861786
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,15,0.011791999638080596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,31,0.01175519973039627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,63,0.011822400242090225
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,127,0.012135999649763108
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,255,0.01828159987926483
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,511,0.022096000611782074
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,1023,0.029864001274108886
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,2047,0.04167200028896332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,1,0.011870399862527848
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,3,0.01188960000872612
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,7,0.011878400295972823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,1023,1.9118511199951171
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,15,0.011873599886894227
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,63,0.01212640032172203
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,127,0.012460800260305405
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,3,0.1170192003250122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,31,0.012083200365304947
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,511,0.03642399907112122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,1023,0.05677279829978943
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,2047,0.10691360235214234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,3,0.0643775999546051
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,1,0.06530560255050659
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,7,0.06442239880561829
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,15,0.06540480256080627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,31,0.0646511971950531
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,127,0.06814560294151306
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,63,0.06584799885749817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,255,0.24331519603729249
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,3,0.011817599833011627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,511,0.4926015853881836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,1,0.017423999309539796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,3,0.017476800084114074
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,7,0.017430399358272553
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,15,0.01748799979686737
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,31,0.017510400712490083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,63,0.01769919991493225
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,127,0.018320000171661376
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,255,0.026454401016235352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,1023,0.9694592475891113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,511,0.05448960065841675
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,2047,0.31127679347991943
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,255,0.0164015993475914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,1,0.2219343900680542
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,3,0.22244000434875488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,7,0.22219998836517335
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,2047,1.9649087905883789
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,2047,1.177945613861084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,15,0.22281439304351808
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,31,0.22301759719848632
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,63,0.2723952054977417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,127,0.4790031909942627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,511,0.9519424438476562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,255,0.940129566192627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,2047,3.9105503082275392
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,1,0.43251519203186034
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,3,0.4321023941040039
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,7,0.4327888011932373
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,511,1.8724399566650392
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,15,0.43436322212219236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,63,0.49701762199401855
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,31,0.43839201927185056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,127,0.9474111557006836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,40,40,128,1,bfloat16,bfloat16,1,0.8600367546081543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,40,40,128,1,bfloat16,bfloat16,15,0.8511039733886718
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,40,40,128,1,bfloat16,bfloat16,3,0.8592880249023438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,40,40,128,1,bfloat16,bfloat16,7,0.8533920288085938
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,1,0.02377759963274002
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,40,40,128,1,bfloat16,bfloat16,255,1.867401695251465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,3,0.02385119944810867
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,40,40,128,1,bfloat16,bfloat16,31,0.8885024070739747
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,15,0.023740799725055696
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,63,0.024120000004768372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,31,0.024281600117683412
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,127,0.025007998943328856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,7,0.02340639978647232
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,255,0.03818880021572113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,511,0.08943359851837158
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,1023,3.8081871032714845
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,1023,0.291644811630249
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,2047,0.5270527839660645
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,40,40,128,1,bfloat16,bfloat16,127,1.890880012512207
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,1023,0.11048640012741089
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,40,40,128,1,bfloat16,bfloat16,1,1.6886783599853517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,40,40,128,1,bfloat16,bfloat16,7,1.6875232696533202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,40,40,128,1,bfloat16,bfloat16,3,1.6885200500488282
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,40,40,128,1,bfloat16,bfloat16,15,1.7561168670654297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,40,40,128,1,bfloat16,bfloat16,31,1.7652751922607421
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,40,40,128,1,bfloat16,bfloat16,63,0.9804495811462403
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,40,40,128,1,bfloat16,bfloat16,63,1.9796527862548827
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,1,0.03535679876804352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,7,0.03535040020942688
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,3,0.0380048006772995
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,15,0.03542400002479553
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,31,0.03835200071334839
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,63,0.0361519992351532
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,255,0.05846719741821289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,127,0.040720000863075256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,40,40,128,1,bfloat16,bfloat16,1,3.52437744140625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,511,0.2602463960647583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,40,40,128,1,bfloat16,bfloat16,3,3.5317569732666017
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,1023,0.5249392032623291
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,1,0.09467200040817261
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,40,40,128,1,bfloat16,bfloat16,7,3.5677104949951173
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,15,0.09496480226516724
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,40,40,128,1,bfloat16,bfloat16,15,3.577897644042969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,2047,0.9881823539733887
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,63,0.09671199917793274
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,3,0.09514880180358887
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,31,0.09804319739341735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,255,0.3824431896209717
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,40,40,128,1,bfloat16,bfloat16,31,3.6935375213623045
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,511,0.7663055896759033
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,3,0.011425600200891495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,1,0.012148799747228623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,7,0.011423999816179276
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,31,0.011452800035476685
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,1023,1.5308496475219726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,127,0.011787199974060058
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,63,0.011617600172758102
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,255,0.015806399285793304
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,1023,0.026035198569297792
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,2047,0.03383679986000061
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,4095,0.05421599745750427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,3,0.011577600240707397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,1,0.011539199948310852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,7,0.011584000289440155
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,31,0.011598400026559829
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,15,0.011646399646997452
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,63,0.0117296002805233
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,255,0.01595200002193451
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,127,0.012076800316572189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,511,0.025931200385093688
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,1023,0.033667200803756715
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,7,0.09480000138282776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,2047,0.05417119860649109
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,2047,3.1285648345947266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,3,0.052724802494049074
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,4095,0.22028160095214844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,1,0.05355679988861084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,15,0.0529695987701416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,127,0.21559998989105225
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,31,0.053127998113632204
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,127,0.05571519732475281
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,63,0.05441920161247253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,7,0.0529744029045105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,255,0.19611519575119019
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,1,0.0118367999792099
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,511,0.4018064022064209
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,3,0.011857599765062333
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,15,0.011883199959993363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,31,0.011902400106191636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,15,0.011713600158691407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,7,0.011777599900960922
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,63,0.012166400253772736
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,1023,0.7894288063049316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,127,0.012644800543785095
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,255,0.01658080071210861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,511,0.021211199462413788
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,511,0.03272159993648529
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,2047,0.21729118824005128
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,1023,0.05288800001144409
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,1,0.17723360061645507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,3,0.17698240280151367
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,4095,0.4018400192260742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,15,0.17665280103683473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,31,0.17789920568466186
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,63,0.2070688009262085
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,2047,1.599995231628418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,127,0.3834624052047729
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,255,0.752942419052124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,4095,6.099335861206055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,1,0.3461472034454346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,511,1.5015328407287598
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,4095,3.138128089904785
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,7,0.34604640007019044
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,3,0.3473695993423462
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,63,0.4020319938659668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,15,0.3479935884475708
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,31,0.3492559909820557
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,127,0.8438112258911132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,7,0.1765984058380127
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,1,0.68787522315979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,1023,3.1241024017333983
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,3,0.6864336013793946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,7,0.6878015995025635
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,15,0.6884287834167481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,255,1.4938591957092284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,1,0.01751679927110672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,31,0.7223423957824707
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,63,0.8458736419677735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,3,0.017558400332927705
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,7,0.017652800679206847
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,31,0.017662400007247926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,63,0.018027199804782866
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,127,0.01891999989748001
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,127,1.5094592094421386
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,255,0.02707040011882782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,511,0.05495679974555969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,1023,0.2344559907913208
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,2047,0.4352848052978516
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,2047,6.242574310302734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,4095,0.7862607955932617
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,32,128,1,bfloat16,bfloat16,255,2.9773263931274414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,32,128,1,bfloat16,bfloat16,1,1.3565967559814454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,32,128,1,bfloat16,bfloat16,3,1.36146879196167
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,32,128,1,bfloat16,bfloat16,15,1.390924835205078
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,15,0.018401600420475006
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,32,128,1,bfloat16,bfloat16,7,1.3534031867980958
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,32,128,1,bfloat16,bfloat16,511,2.972110366821289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,32,128,1,bfloat16,bfloat16,31,1.4325695991516114
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,32,128,1,bfloat16,bfloat16,63,1.5932047843933106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,1,0.02924799919128418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,3,0.02924000024795532
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,7,0.029278400540351867
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,15,0.029342401027679443
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,31,0.02938719987869263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,32,128,1,bfloat16,bfloat16,3,2.7811168670654296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,63,0.029879999160766602
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,127,0.03133119940757752
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,32,128,1,bfloat16,bfloat16,1,2.7789615631103515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,255,0.04752320051193237
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,32,128,1,bfloat16,bfloat16,127,3.003246307373047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,511,0.2121664047241211
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,1023,0.4057616233825684
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,1,0.07212640047073364
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,32,128,1,bfloat16,bfloat16,15,2.8342815399169923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,7,0.0722495973110199
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,31,0.07247359752655029
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,3,0.07329599857330323
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,63,0.07332800030708313
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,127,0.08802400231361389
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,15,0.07506880164146423
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,255,0.29074559211730955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,4095,1.5791263580322266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,511,0.5830128192901611
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,32,128,1,bfloat16,bfloat16,63,3.3565696716308593
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,1,0.011275199800729751
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,7,0.011296000331640244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,1023,1.1640144348144532
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,15,0.011308799684047698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,63,0.011425600200891495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,127,0.011627200245857238
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,3,0.012078399956226348
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,255,0.015532800555229187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,31,0.011419200152158738
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,1023,0.022019200026988983
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,511,0.019079999625682832
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,4095,0.04517279863357544
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,3,0.011377599835395814
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,7,0.01138719990849495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,2047,2.34976806640625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,15,0.011380799859762192
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,32,128,1,bfloat16,bfloat16,31,2.9392160415649413
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,63,0.011555200070142746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,2047,0.7931168079376221
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,255,0.01571040004491806
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,31,0.011392000317573547
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,2047,0.06330720186233521
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,127,0.012265600264072418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,1,0.041894400119781496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,4095,0.1799023985862732
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,3,0.0418832004070282
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,7,0.041308799386024476
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,15,0.04193120002746582
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,31,0.04146719872951508
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,63,0.04276799857616424
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,127,0.04365600049495697
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,32,128,1,bfloat16,bfloat16,7,2.876900863647461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,255,0.07762560248374939
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,2047,0.030419200658798218
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,1,0.012135999649763108
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,511,0.30606880187988283
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,1023,0.039401599764823915
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,1023,0.5915311813354492
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,3,0.011507199704647064
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,1,0.012223999947309494
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,15,0.011558400094509124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,63,0.011788800358772278
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,31,0.012080000340938568
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,511,0.027809599041938783
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,2047,1.1826111793518066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,511,0.03669599890708923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,255,0.016443200409412384
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,1023,0.055731201171875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,2047,0.1371232032775879
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,1,0.1364527940750122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,3,0.13670560121536254
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,7,0.13677120208740234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,4095,0.31512160301208497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,15,0.13713279962539673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,31,0.13725119829177856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,4095,2.283943939208984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,63,0.14970879554748534
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,127,0.2895136117935181
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,255,0.5689104080200196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,4095,4.73852310180664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,1,0.2661407947540283
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,7,0.26656160354614256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,511,1.2264752388000488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,3,0.26604158878326417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,7,0.011596799641847611
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,31,0.26789920330047606
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,63,0.32558560371398926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,127,0.5697648048400878
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,1023,2.3073423385620115
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,255,1.1268400192260741
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,1,0.5192768096923828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,3,0.5194191932678223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,15,0.5213856220245361
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,7,0.5223504066467285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,31,0.55033278465271
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,63,0.6037487983703613
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,15,0.2672528028488159
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,127,0.012129600346088409
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,24,24,128,1,bfloat16,bfloat16,511,2.339422416687012
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,1,0.01762239933013916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,3,0.017235200107097625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,2047,4.6910560607910154
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,31,0.01772480010986328
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,7,0.017580799758434296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,63,0.01757120043039322
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,15,0.01748320013284683
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,255,0.02666719853878021
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,127,1.138542366027832
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,511,0.054369598627090454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,1023,0.13624000549316406
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,127,0.01836320012807846
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,2047,0.3381871938705444
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,4095,0.5892303943634033
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,24,24,128,1,bfloat16,bfloat16,255,2.244887924194336
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,24,24,128,1,bfloat16,bfloat16,7,1.0263104438781738
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,24,24,128,1,bfloat16,bfloat16,3,1.0231504440307617
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,24,24,128,1,bfloat16,bfloat16,15,1.0351072311401368
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,24,24,128,1,bfloat16,bfloat16,1,1.02423038482666
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,24,24,128,1,bfloat16,bfloat16,31,1.0969023704528809
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,24,24,128,1,bfloat16,bfloat16,63,1.1969152450561524
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,24,24,128,1,bfloat16,bfloat16,1,2.064107131958008
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,24,24,128,1,bfloat16,bfloat16,3,2.034903907775879
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,1,0.023689599335193635
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,24,24,128,1,bfloat16,bfloat16,7,2.0949296951293945
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,24,24,128,1,bfloat16,bfloat16,127,2.2508111953735352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,7,0.023771199584007262
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,15,0.023923200368881226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,3,0.023363199830055238
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,63,0.02444639950990677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,127,0.02569119930267334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,24,24,128,1,bfloat16,bfloat16,31,2.1864736557006834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,255,0.03694559931755066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,511,0.13528800010681152
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,31,0.023614400625228883
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,1023,0.31100800037384035
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,1,0.05365440249443054
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,31,0.05397599935531616
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,2047,0.6026576042175293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,63,0.054711997509002686
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,255,0.19517439603805542
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,4095,1.1572815895080566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,24,24,128,1,bfloat16,bfloat16,63,2.4818368911743165
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,127,0.05783680081367493
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,511,0.40125441551208496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,1023,0.7872352123260498
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,24,24,128,1,bfloat16,bfloat16,15,2.106540870666504
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,1,0.011054400354623795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,7,0.011099199950695037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,15,0.011073599755764007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,31,0.011129599809646607
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,3,0.013752000033855438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,63,0.013583999872207642
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,3,0.05356320142745972
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,255,0.015475200116634369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,127,0.012511999905109405
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,511,0.017126399278640746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,1023,0.021367999911308288
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,2047,0.025787198543548585
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,4095,0.03447360098361969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,8191,0.05392959713935852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,1,0.013422399759292603
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,4095,3.1030048370361327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,3,0.011270400136709213
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,7,0.01265760064125061
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,31,0.011326400190591812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,127,0.011582399904727935
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,63,0.011660800129175187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,15,0.01120000034570694
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,255,0.015539200603961944
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,511,0.02144480049610138
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,2047,0.03385440111160278
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,1023,0.02585279941558838
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,4095,0.05411520004272461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,1,0.029558399319648744
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,8191,0.22019519805908203
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,3,0.029840001463890077
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,7,0.029691201448440552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,31,0.030025601387023926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,15,0.029600000381469725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,63,0.030241599678993224
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,255,0.04772000014781952
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,2047,1.5951696395874024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,7,0.05411199927330017
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,511,0.2294719934463501
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,2047,0.8636303901672363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,8191,6.113065719604492
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,1,0.011336000263690948
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,7,0.011313600093126297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,15,0.01130559965968132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,4095,1.527996826171875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,31,0.011396799981594086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,3,0.011316800117492675
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,127,0.011776000261306763
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,255,0.015703999996185304
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,511,0.025723201036453248
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,63,0.01159999966621399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,2047,0.053487998247146604
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,1023,0.03356159925460815
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,4095,0.23561279773712157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,127,0.03195840120315552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,8191,0.40300641059875486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,7,0.09647200107574463
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,3,0.09560959935188293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,15,0.09625759720802307
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,63,0.09743679761886596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,1023,0.4038064002990723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,15,0.05448960065841675
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,8191,2.990430450439453
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,127,0.1941424012184143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,255,0.3826591968536377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,1,0.1795904040336609
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,3,0.17976800203323365
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,511,0.7651343822479248
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,7,0.17995519638061525
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,31,0.1803871989250183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,15,0.17980159521102906
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,1023,1.5299983978271485
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,127,0.3813744068145752
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,255,0.7532464027404785
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,1,0.09554240107536316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,31,0.09667199850082397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,3,0.3533263921737671
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,2047,3.1213056564331056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,511,1.503371238708496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,7,0.3535264015197754
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,15,0.3543600082397461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,63,0.4094111919403076
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,31,0.3569999933242798
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,1,0.011444800347089768
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,127,0.754256010055542
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,63,0.20883519649505616
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,15,0.01144160032272339
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,7,0.011822400242090225
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,31,0.0115167997777462
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,127,0.012203200161457062
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,255,1.4960335731506347
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,16,128,1,bfloat16,bfloat16,1023,3.0309871673583983
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,511,0.03209599852561951
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,1023,0.052323198318481444
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,2047,0.2343951940536499
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,1,0.3523263931274414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,4095,6.078332901000977
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,4095,0.4405824184417725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,1,0.6923808097839356
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,8191,0.7710959911346436
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,7,0.6969071865081787
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,16,128,1,bfloat16,bfloat16,511,2.975254440307617
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,3,0.6922319889068603
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,15,0.6932960033416748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,3,0.011479999870061874
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,31,0.7238639831542969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,63,0.8019120216369628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,255,0.016212800145149232
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,63,0.012044800072908401
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,16,128,1,bfloat16,bfloat16,1,1.3687984466552734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,127,1.584067153930664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,16,128,1,bfloat16,bfloat16,7,1.4232912063598633
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,16,128,1,bfloat16,bfloat16,3,1.3709471702575684
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,3,0.017342400550842286
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,1,0.01807200014591217
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,16,128,1,bfloat16,bfloat16,15,1.4036704063415528
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,7,0.0173552006483078
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,16,128,1,bfloat16,bfloat16,31,1.428116798400879
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,63,0.018163199722766876
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,255,0.02733440101146698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,31,0.018216000497341157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,127,0.01907680034637451
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,511,0.05536320209503174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,16,128,1,bfloat16,bfloat16,63,1.608286476135254
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,1023,0.21736159324645996
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,16,128,1,bfloat16,bfloat16,255,3.257923126220703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,2047,0.4138800144195557
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,1,0.042345601320266726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,4095,0.7820816040039062
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,15,0.04251199960708618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,7,0.04227199852466583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,127,0.04494880139827728
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,16,128,1,bfloat16,bfloat16,127,2.968035125732422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,255,0.07723360061645508
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,31,0.04241119921207428
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,8191,1.559712028503418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,15,0.017873600125312805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,1023,0.5915071964263916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,511,0.33439040184020996
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,1,0.011244799941778183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,3,0.011211200058460236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,7,0.011219199746847153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,2047,1.1816271781921386
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,15,0.011188799887895584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,31,0.011224000155925751
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,255,0.01542080044746399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,127,0.011476799845695496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,511,0.016956800222396852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,2047,0.022735999524593355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,63,0.011351999640464783
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,1023,0.021886399388313292
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,4095,0.03055039942264557
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,1,0.011179199814796448
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,3,0.011326400190591812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,15,0.01130400002002716
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,8191,0.04300000071525574
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,7,0.011241599917411804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,31,0.011360000073909759
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,3,0.042028799653053284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,63,0.042998400330543515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,63,0.011468800157308579
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,255,0.015471999347209931
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,127,0.011556799709796905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,1023,0.02292640060186386
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,2047,0.030990400910377504
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,511,0.018780800700187682
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,1,0.024115200340747833
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,3,0.023895999789237975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,4095,2.389262390136719
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,31,0.023979200422763823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,8191,0.17100800275802613
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,15,0.02402399927377701
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,7,0.023953600227832793
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,255,0.03777439892292023
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,511,0.13407039642333984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,63,0.024369600415229797
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,1,0.011324799805879592
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,127,0.025699201226234435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,1023,0.3109247922897339
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,3,0.011360000073909759
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,31,0.011423999816179276
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,7,0.011430399864912033
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,63,0.011505600064992905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,15,0.011601600050926208
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,127,0.011830399930477142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,511,0.028519999980926514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,2047,0.6598112106323242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,1023,0.040375998616218566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,2047,0.06407039761543273
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,4095,1.1595215797424316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,1,0.07237759828567505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,7,0.07269440293312072
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,3,0.07250720262527466
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,15,0.07235199809074402
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,8191,0.41275677680969236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,8191,4.493507385253906
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,31,0.07297919988632202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,63,0.07402560114860535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,127,0.09202719926834106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,4095,0.05179679989814758
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,8191,2.2664031982421875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,255,0.32038719654083253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,511,0.5837567806243896
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,1,0.1383072018623352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,3,0.13827040195465087
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,7,0.13933759927749634
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,15,0.1387120008468628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,31,0.13988159894943236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,1023,1.1570015907287599
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,63,0.1513983964920044
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,127,0.29016799926757814
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,255,0.015695999562740325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,1,0.2698240041732788
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,255,0.5969967842102051
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,3,0.2700000047683716
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,4095,0.1799183964729309
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,2047,2.35853271484375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,7,0.270084810256958
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,15,0.27082560062408445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,511,1.139806365966797
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,127,0.5700064182281495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,63,0.3329936027526855
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,3,0.011584000289440155
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,1,0.011526399850845337
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,7,0.011529599875211715
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,15,0.011593600362539291
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,31,0.011540800333023071
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,255,1.1312255859375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,63,0.011791999638080596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,12,12,128,1,bfloat16,bfloat16,1023,2.2846879959106445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,511,0.037231999635696414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,127,0.012118399888277055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,255,0.016102400422096253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,2047,0.13375040292739868
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,1023,0.05616160035133362
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,4095,4.5658622741699215
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,4095,0.3166975975036621
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,1,0.5237055778503418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,511,2.2476783752441407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,8191,0.5923088073730469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,15,0.5254992008209228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,7,0.5250112056732178
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,31,0.5483151912689209
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,12,12,128,1,bfloat16,bfloat16,31,0.27191040515899656
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,63,0.6536896228790283
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,127,1.1244256019592285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,12,12,128,1,bfloat16,bfloat16,1,1.0315936088562012
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,12,12,128,1,bfloat16,bfloat16,7,1.0346639633178711
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,12,12,128,1,bfloat16,bfloat16,3,1.0311440467834472
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,1,0.017340800166130065
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,3,0.01740480065345764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,7,0.017479999363422392
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,12,12,128,1,bfloat16,bfloat16,15,1.0450592041015625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,15,0.017369599640369417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,63,0.017692799866199493
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,31,0.01744000017642975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,127,0.01841440051794052
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,255,2.247219276428223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,255,0.027039998769760133
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,511,0.054385602474212646
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,12,12,128,1,bfloat16,bfloat16,63,1.2705151557922363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,2047,0.31531200408935545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,12,12,128,1,bfloat16,bfloat16,3,0.5235775947570801
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,1023,0.14872159957885742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,4095,0.5891712188720704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,1,0.030048000812530517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,31,0.03025439977645874
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,63,0.03052000105381012
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,15,0.02982879877090454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,12,12,128,1,bfloat16,bfloat16,127,2.2369775772094727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,127,0.031918400526046754
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,255,0.04785760045051575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,8191,1.1471983909606933
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,1023,0.4060224056243896
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,511,0.2298640012741089
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,2047,0.7933279991149902
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,1,0.01096159964799881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,7,0.010976000130176545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,3,0.01085119992494583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,15,0.010992000252008438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,12,12,128,1,bfloat16,bfloat16,31,1.095456027984619
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,31,0.010902400314807891
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,255,0.015059199929237366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,63,0.011099199950695037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,511,0.015409600734710694
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,2047,0.02088959962129593
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,4095,0.025446400046348572
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,4095,1.5356911659240722
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,127,0.01138240024447441
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,8191,0.0340175986289978
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,3,0.010907199978828431
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,1,0.011161600053310395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,7,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,16383,0.05399199724197388
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,15,0.01109279990196228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,31,0.010903999954462052
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,3,0.03020319938659668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,255,0.015134400129318238
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,127,0.011347199976444244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,63,0.011140800267457961
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,1023,0.020670400559902193
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,511,0.017108799517154695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,2047,0.025748801231384278
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,8191,0.05304480195045471
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,4095,0.03377920091152191
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,1,0.017628799378871917
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,3,0.017475199699401856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,7,0.017740799486637114
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,16383,0.21999518871307372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,15,0.017526400089263917
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,31,0.017771199345588684
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,127,0.018884800374507904
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,8191,3.005299186706543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,255,0.027212798595428467
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,63,0.018268799781799315
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,511,0.05521439909934998
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,2047,0.41355037689208984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,1023,0.2331007957458496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,1,0.010943999886512757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,4095,0.7835072040557861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,3,0.010886400192975997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,1023,0.020423999428749083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,15,0.010918399691581726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,31,0.011060799658298492
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,127,0.011367999762296677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,63,0.011032000184059143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,255,0.015316799283027649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,8191,1.5258912086486816
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,1023,0.025755199790000915
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,511,0.02084160000085831
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,4095,0.05398079752922058
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,7,0.030083200335502623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,2047,0.03330560028553009
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,8191,0.23645761013031005
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,3,0.05394560098648071
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,1,0.05437759757041931
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,16383,0.4046304225921631
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,7,0.054129600524902344
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,15,0.05405279994010925
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,16383,5.9527534484863285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,31,0.054799997806549074
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,127,0.056806397438049314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,255,0.19595520496368407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,511,0.441212797164917
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,1023,0.7909791946411133
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,16383,3.019403266906738
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,1,0.09691519737243652
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,3,0.09701279997825622
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,7,0.011168000102043153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,2047,1.6066335678100585
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,15,0.09697279930114747
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,31,0.0972432017326355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,63,0.09846879839897156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,127,0.2158207893371582
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,255,0.3843456029891968
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,63,0.055420798063278195
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,4095,3.1108640670776366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,511,0.7705247879028321
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,1,0.18258399963378907
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,15,0.18276959657669067
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,3,0.18216480016708375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,7,0.182425594329834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,31,0.18336160182952882
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,63,0.2275264024734497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,1023,1.6045488357543944
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,127,0.38292319774627687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,7,0.0969103991985321
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,1,0.011043199896812439
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,7,0.011105599999427795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,3,0.01106399968266487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,255,0.7580016136169434
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,31,0.011068800091743469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,127,0.01154559999704361
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,255,0.01539520025253296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,63,0.011472000181674958
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,511,0.025547200441360475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,1023,0.033406400680541994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,2047,0.05359359979629517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,8,128,1,bfloat16,bfloat16,2047,3.1310911178588867
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,4095,0.2204591989517212
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,511,1.6223312377929688
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,1,0.3542543888092041
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,7,0.3550944089889526
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,3,0.35269761085510254
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,8191,6.2912353515625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,15,0.3545952081680298
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,31,0.3550559997558594
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,8,128,1,bfloat16,bfloat16,1023,3.088630485534668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,63,0.4286320209503174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,15,0.01114879995584488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,127,0.7520415782928467
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,3,0.6925471782684326
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,1,0.6930367946624756
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,7,0.6963712215423584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,255,1.4991776466369628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,8191,0.40399680137634275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,15,0.6939472198486328
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,31,0.7282591819763183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,1,0.01125119999051094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,7,0.01122559979557991
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,63,0.8002400398254395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,15,0.011310400068759918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,16383,0.857487964630127
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,63,0.01149120032787323
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,3,0.011284799873828888
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,255,0.01600320041179657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,127,0.012140800058841706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,31,0.012028799951076507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,511,0.0320576012134552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,127,1.4854576110839843
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,8,128,1,bfloat16,bfloat16,511,2.9885120391845703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,2047,0.23513441085815429
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,4095,0.4024384021759033
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,8191,0.7733888149261474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,1,0.01793919950723648
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,7,0.01818400025367737
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,63,0.0184688001871109
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,31,0.01794400066137314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,127,0.0192671999335289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,15,0.01788640022277832
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,255,0.02720000147819519
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,511,0.056302398443222046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,16383,1.5747759819030762
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,1023,0.21587200164794923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,2047,0.4497647762298584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,4095,0.7899504184722901
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,1,0.010907199978828431
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,1023,0.051811200380325315
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,3,0.010902400314807891
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,15,0.010950399935245514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,31,0.010956799983978272
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,8191,1.5290863990783692
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,63,0.011022400110960007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,255,0.015147200226783753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,511,0.016174399852752687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,3,0.018275199830532073
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,1023,0.020201599597930907
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,4095,0.021796800196170807
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,127,0.01117599979043007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,8191,0.026144000887870788
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,2047,0.02131839990615845
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,32767,0.05445759892463684
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,16383,0.034657600522041324
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,1,0.01281919926404953
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,3,0.010942400246858597
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,15,0.01154400035738945
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,31,0.010985600203275681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,7,0.010891199856996537
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,127,0.011212799698114395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,63,0.012636800110340119
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,16383,3.0147823333740233
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,511,0.016339200735092162
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,255,0.015084800124168397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,1023,0.021495999395847322
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,8191,0.03419840037822723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,2047,0.021635200083255767
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,16383,0.05457119941711426
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,1,0.011604800075292587
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,32767,0.22104959487915038
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,3,0.013102400302886962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,7,0.011662399768829346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,15,0.011606399714946748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,31,0.012960000336170197
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,63,0.011963199824094772
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,255,0.016788800060749055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,511,0.032286399602890016
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,1023,0.053092801570892335
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,7,0.010876800119876861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,2047,0.23640639781951905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,4095,0.40194082260131836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,8191,0.7767663955688476
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,1,0.010868799686431885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,8,128,1,bfloat16,bfloat16,255,2.984929656982422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,3,0.010819199681282043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,4095,0.026230400800704955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,15,0.01088479980826378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,63,0.010995200276374817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,31,0.011022400110960007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,127,0.011212799698114395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,255,0.015137599408626556
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,32767,5.957732772827148
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,511,0.017420800030231477
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,1023,0.021563200652599333
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,4095,0.03421919941902161
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,8191,0.05426080226898193
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,127,0.012332800030708312
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,7,0.01249760016798973
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,1,0.030287998914718627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,16383,0.22084639072418213
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,7,0.030342400074005127
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,3,0.030769601464271545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,32767,0.4046351909637451
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,127,0.03257119953632355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,63,0.03142080008983612
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,255,0.048851200938224794
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,32767,3.011756706237793
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,1023,0.4076496124267578
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,511,0.22992320060729982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,2047,0.8638575553894043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,1,0.055088001489639285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,16383,1.534995174407959
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,4095,1.5398176193237305
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,2047,0.026169601082801818
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,3,0.055169600248336795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,31,0.055251199007034305
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,31,0.03039039969444275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,63,0.05600960254669189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,127,0.0582863986492157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,255,0.19628479480743408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,511,0.4028016090393066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,8191,3.0006879806518554
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,1023,0.7915040016174316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,1,0.09897760152816773
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,3,0.09905599951744079
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,7,0.09926720261573792
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,15,0.09912639856338501
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,2047,1.6010719299316407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,31,0.09943199753761292
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,15,0.03039039969444275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,7,0.05516800284385681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,15,0.055062401294708255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,127,0.19610400199890138
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,63,0.10123039484024048
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,255,0.385263991355896
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,1,0.010921599715948105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,7,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,3,0.010911999642848969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,15,0.010923200100660325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,511,0.7714672088623047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,31,0.010942400246858597
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,4,128,1,bfloat16,bfloat16,4095,3.1163536071777345
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,255,0.01518079936504364
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,1023,0.026438400149345398
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,127,0.011406400054693223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,2047,0.03419199883937836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,16383,5.9416961669921875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,511,0.021641600131988525
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,4095,0.0543503999710083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,1023,1.5401552200317383
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,8191,0.22291359901428223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,1,0.18247519731521605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,7,0.1826032042503357
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,3,0.18208160400390624
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,15,0.1827008008956909
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,16383,0.4451871871948242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,31,0.18316639661788942
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,63,0.22476000785827638
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,127,0.3833071947097778
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,32767,0.9846672058105469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,4,128,1,bfloat16,bfloat16,2047,3.135188865661621
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,1,0.35297279357910155
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,255,0.8417920112609864
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,3,0.35286240577697753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,15,0.35328800678253175
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,7,0.3547168016433716
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,31,0.3547152042388916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,511,1.5089344024658202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,63,0.011113599687814713
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,63,0.4387712001800537
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,1,0.011028800159692764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,7,0.01101600006222725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,127,0.7518352031707763
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,15,0.011032000184059143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,31,0.01104160025715828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,255,0.015408000349998474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,127,0.0114656001329422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,511,0.026305601000785828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,63,0.011241599917411804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,1023,0.03391200006008148
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,2047,0.054478400945663454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,4095,0.22244958877563475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,8191,0.4051504135131836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,255,1.6638320922851562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,4,128,1,bfloat16,bfloat16,1023,3.05328311920166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,1,0.012049599736928939
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,16383,0.7783023834228515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,31,0.01204800009727478
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,127,0.012740799784660339
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,255,0.016748799383640288
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,3,0.012055999785661697
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,511,0.033011201024055484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,1023,0.05305280089378357
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,2047,0.21519200801849364
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,32767,1.5155839920043945
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,4095,0.3924495935440063
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,4,128,1,bfloat16,bfloat16,511,3.0078880310058596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,3,0.011088000237941742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,8191,0.7523680210113526
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,1,0.012111999839544297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,7,0.01120320037007332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,3,0.010705599933862687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,31,0.010769599676132202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,63,0.010843200236558914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,16383,1.4792559623718262
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,127,0.011079999804496764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,511,0.016195200383663177
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,1023,0.020151999592781068
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,2047,0.020259200036525725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,4095,0.020947200059890748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,255,0.015011200308799743
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,8191,0.02311040014028549
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,32767,0.03595199882984161
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,16383,0.02776640057563782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,65535,0.05569120049476624
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,1,0.0108255997300148
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,7,0.012270399928092956
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,3,0.010768000036478043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,15,0.010828799754381179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,7,0.011035200208425522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,31,0.010753600299358368
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,127,0.011084800213575363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,63,0.011211200058460236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,255,0.014955200254917145
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,32767,2.901238441467285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,1023,0.020326399803161622
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,4095,0.02163040041923523
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,511,0.017071999609470367
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,8191,0.0274399995803833
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,2047,0.02107519954442978
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,16383,0.036350399255752563
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,32767,0.055796802043914795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,1,0.011617600172758102
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,3,0.011377599835395814
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,15,0.011340799927711486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,15,0.01133119985461235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,7,0.011598400026559829
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,65535,0.2287343978881836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,31,0.01138240024447441
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,127,0.012084800004959106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,63,0.011878400295972823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,511,0.027169600129127502
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,255,0.015806399285793304
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,1023,0.03564000129699707
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,2047,0.05572159886360169
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,4095,0.2170367956161499
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,8191,0.42588157653808595
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,16383,0.7584095954895019
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,15,0.0123648002743721
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,1,0.010903999954462052
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,3,0.010875199735164643
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,15,0.010967999696731567
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,7,0.012011200189590454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,31,0.011073599755764007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,63,0.011007999628782272
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,255,0.015132799744606018
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,127,0.01196800023317337
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,511,0.016667200624942778
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,1023,0.021347199380397797
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,4095,0.026281601190567015
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,8191,0.03598879873752594
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,2047,0.022078399360179902
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,16383,0.05567200183868408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,1,0.01842560023069382
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,3,0.01844319999217987
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,7,0.018488000333309173
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,15,0.018467199802398682
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,31,0.0185248002409935
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,65535,0.42839841842651366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,63,0.0188400000333786
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,127,0.019814400374889372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,65535,2.9115999221801756
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,511,0.05673760175704956
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,255,0.028112000226974486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,1023,0.22537760734558104
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,63,0.01228640004992485
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,4095,0.7586448192596436
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,65535,5.864838409423828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,32767,1.4754079818725585
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,1,0.03140319883823395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,3,0.0315311998128891
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,7,0.03148959875106812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,32767,0.2149199962615967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,31,0.031585600972175595
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,8191,1.6054927825927734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,15,0.03161439895629883
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,63,0.03205280005931854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,127,0.03349120020866394
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,255,0.04997600018978119
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,511,0.21779680252075195
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,1023,0.39676001071929934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,1,0.05757279992103577
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,3,0.05761759877204895
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,2047,0.4011839866638184
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,16383,2.9019487380981444
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,7,0.05779520273208618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,15,0.05768640041351318
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,127,0.060915201902389526
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,2047,0.8310576438903808
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,255,0.19428639411926268
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,63,0.05845440030097961
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,4095,1.4920767784118651
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,511,0.39466400146484376
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,3,0.010915199667215348
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,1023,0.8278351783752441
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,7,0.010953599959611893
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,31,0.010945600271224976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,63,0.011052799969911575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,127,0.011556799709796905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,15,0.010857599973678588
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,255,0.015083199739456177
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,1023,0.021817600727081297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,511,0.017723199725151063
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,2047,1.5521183967590333
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,2,128,1,bfloat16,bfloat16,8191,2.9212080001831056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,4095,0.03439199924468994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,8191,0.05583999752998352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,16383,0.2298367977142334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,32767,0.4279615879058838
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,32767,5.74903678894043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,31,0.05776960253715515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,1,0.09911839962005616
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,15,0.09923359751701355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,7,0.09947680234909058
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,65535,1.0086655616760254
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,2,128,1,bfloat16,bfloat16,4095,3.184721565246582
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,3,0.09885280132293701
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,63,0.10127999782562255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,127,0.21335840225219727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,1,0.010868799686431885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,255,0.41063680648803713
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,1,0.1824671983718872
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,7,0.1828719973564148
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,511,0.8236127853393554
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,3,0.1820207953453064
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,2047,0.02632800042629242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,15,0.18274879455566406
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,63,0.22907679080963134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,31,0.18312640190124513
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,1023,1.5002575874328614
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,127,0.3798799991607666
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,255,0.7370863914489746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,3,0.010939200222492219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,1,0.011151999980211259
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,15,0.010952000319957734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,63,0.011081600189208984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,31,0.011137600243091583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,7,0.010907199978828431
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,127,0.011313600093126297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,1023,0.026679998636245726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,255,0.015360000729560851
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,2047,0.034625598788261415
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,31,0.09957919716835022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,8191,0.2170480012893677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,4095,0.05462560057640076
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,511,1.5282575607299804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,16383,0.3952784061431885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,2,128,1,bfloat16,bfloat16,2047,3.0369184494018553
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,32767,0.8881952285766601
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,1,0.011740799993276596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,31,0.01180799975991249
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,63,0.011924800276756287
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,15,0.011751999706029892
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,65535,1.7316303253173828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,127,0.012201599776744843
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,2,128,1,bfloat16,bfloat16,1023,3.183344078063965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,255,0.016147199273109435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,1023,0.0354559987783432
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,4095,0.21672959327697755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,511,0.02768639922142029
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,8191,0.40044798851013186
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,511,0.021967999637126923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,16383,0.768990421295166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,1,0.01072319969534874
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,3,0.010734400153160096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,7,0.010766399651765823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,3,0.012107200175523757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,15,0.010856000334024429
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,63,0.01091040000319481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,31,0.011668799817562104
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,127,0.011083199828863143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,32767,1.4933792114257813
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,511,0.016457599401474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,255,0.01530880033969879
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,2047,0.020473599433898926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,4095,0.02107200026512146
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,1023,0.02035360038280487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,32767,0.031201601028442383
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,8191,0.022443200647830962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,65535,0.04252960085868836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,1,0.010699199885129929
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,131071,0.06270880103111268
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,7,0.010735999792814255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,31,0.010737600177526474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,3,0.01085280030965805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,65535,2.930683135986328
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,63,0.010870400071144103
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,15,0.012036799639463424
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,127,0.011108800023794174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,511,0.01637759953737259
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,2047,0.05592160224914551
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,1023,0.020721599459648132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,2047,0.020584000647068022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,4095,0.021038399636745454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,8191,0.023558400571346283
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,255,0.014990399777889251
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,16383,0.030859199166297913
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,65535,0.05886560082435608
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,1,0.01120000034570694
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,7,0.01120320037007332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,15,0.011268799751996994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,31,0.011345600336790084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,127,0.011588799953460693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,131071,0.2365648031234741
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,63,0.011363200098276138
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,255,0.015532800555229187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,1023,0.02709920108318329
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,511,0.022724799811840057
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,2047,0.03519200086593628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,7,0.012100800126791
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,4095,0.05508319735527038
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,16383,0.02621760070323944
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,8191,0.23539841175079346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,131071,5.810793685913086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,16383,0.44271202087402345
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,32767,0.8964927673339844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,1,0.010811199992895126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,3,0.01114879995584488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,15,0.010831999778747558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,31,0.011030399799346923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,7,0.01093600019812584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,63,0.01085119992494583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,127,0.011323200166225433
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,255,0.015059199929237366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,511,0.016543999314308167
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,65535,1.7453039169311524
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,3,0.012932799756526947
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,1023,0.02080480009317398
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,4095,0.021848000586032867
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,16383,0.03951680064201355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,8191,0.02757120132446289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,32767,0.058847999572753905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,1,0.012563200294971466
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,3,0.012692800164222718
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,65535,0.23583359718322755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,131071,0.4007120132446289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,7,0.012556800246238708
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,31,0.01255040019750595
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,127,0.013638399541378021
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,63,0.013072000443935394
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,255,0.017367999255657195
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,511,0.033958399295806886
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,1023,0.054339200258255005
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,4095,0.3959728002548218
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,2047,0.23141279220581054
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,131071,3.458332824707031
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,8191,0.7610943794250489
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,32767,0.03922080099582672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,1,0.01956160068511963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,3,0.019457599520683287
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,15,0.019457599520683287
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,2047,0.021196800470352172
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,7,0.019524799287319185
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,31,0.01960960030555725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,63,0.020078399777412416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,127,0.020916800200939178
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,255,0.02895359992980957
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,511,0.059355199337005615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,16383,1.5040608406066895
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,1023,0.21276960372924805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,2047,0.4054351806640625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,15,0.012731200456619263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,3,0.03405439853668213
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,1,0.033843201398849485
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,7,0.034134399890899655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,15,0.033878400921821594
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,31,0.03426240086555481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,4095,0.7702144145965576
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,63,0.0347104012966156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,127,0.03619999885559082
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,255,0.05277760028839111
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,511,0.21327199935913085
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,8191,1.4956111907958984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,1023,0.40064477920532227
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,32767,3.093659210205078
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,1,0.011070399731397628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,7,0.011032000184059143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,3,0.010830400139093399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,63,0.010974399745464325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,127,0.011128000169992446
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,255,0.015065599977970124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,511,0.016704000532627106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,1023,0.02126079946756363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,4095,1.5082256317138671
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,2047,0.02182080000638962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,1,1,128,1,bfloat16,bfloat16,16383,2.9423152923583986
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,8191,0.035964798927307126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,4095,0.026374399662017822
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,32767,0.21922240257263184
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,65535,0.4407343864440918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,131071,0.7630303859710693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,1,0.05755360126495361
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,65535,5.953532791137695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,7,0.057792001962661745
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,8191,3.034003257751465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,31,0.05775359869003296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,3,0.05772960186004639
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,1,1,128,1,bfloat16,bfloat16,2047,0.7786608219146729
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,127,0.060843199491500854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,15,0.011022400110960007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,15,0.05756319761276245
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,63,0.05863040089607239
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,31,0.010831999778747558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,1,0.09901919960975647
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,3,0.09908639788627624
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,511,0.40153918266296384
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,15,0.09920480251312255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,7,0.09938719868659973
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,31,0.0997759997844696
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,1023,0.778115177154541
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,127,0.1950063943862915
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,16383,0.05906559824943543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,255,0.3912960052490234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,2047,1.5565391540527345
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,511,0.7571375846862793
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,1,0.011046399921178817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,7,0.011017599701881408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,3,0.010844799876213073
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,15,0.0110895998775959
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,31,0.010891199856996537
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,63,0.01098719984292984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,1023,1.5076607704162597
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,127,0.011182399839162827
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,255,0.015072000026702882
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,255,0.19874240159988404
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,2047,0.026374399662017822
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,511,0.017894400656223296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,4095,0.03504480123519897
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,8191,0.055638402700424194
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,1,1,128,1,bfloat16,bfloat16,4095,3.344833755493164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,16383,0.23807840347290038
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,32767,0.40200319290161135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,1,0.01284160017967224
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,63,0.10169600248336792
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,3,0.012647999823093415
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,7,0.012734399735927581
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,65535,0.9314096450805665
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,31,0.012814399600028992
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,15,0.012768000364303589
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,63,0.01286720037460327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,255,0.016950400173664094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,1,1,128,1,bfloat16,bfloat16,2047,3.0469200134277346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,511,0.03138239979743958
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,1023,0.04027040004730224
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,127,0.012992000579833985
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,3,0.012644800543785095
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,1,128,1,bfloat16,bfloat16,2047,0.061592000722885135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,7,0.012716799974441528
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,31,0.012697599828243256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,15,0.012647999823093415
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,1,0.012612800300121307
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,63,0.012828800082206725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,127,0.013238400220870972
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,255,0.017124800384044646
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,511,0.03400799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,1,0.018129600584506987
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,3,0.01807200014591217
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,131071,2.0251136779785157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,15,0.018214400112628936
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,7,0.018027199804782866
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,2047,0.21941280364990234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,2,128,1,bfloat16,bfloat16,1023,0.05574880242347717
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,127,0.01950400024652481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,255,0.027432000637054442
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,511,0.0571615993976593
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,1023,0.021836799383163453
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,63,0.01849440038204193
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,1,0.029811200499534608
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,7,0.029788801074028017
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,3,0.029854398965835572
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,63,0.030559998750686646
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,127,0.03208479881286621
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,2047,0.42085919380187986
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,1023,0.23957281112670897
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,255,0.04810880124568939
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,1,0.01133280023932457
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,511,0.22085919380187988
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,3,0.011307200044393539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,15,0.01143999993801117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,1023,0.413372802734375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,31,0.011416000127792359
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,7,0.011315199732780456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,127,0.011643200367689132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,255,0.015515199303627015
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,511,0.017049600183963776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,1023,0.021172800660133363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,1,0.010940799862146378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,63,0.011532799899578094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,1,128,1,bfloat16,bfloat16,2047,0.021540799736976625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,3,0.010891199856996537
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,7,0.011028800159692764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,15,0.010967999696731567
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,31,0.010979200154542923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,63,0.01114879995584488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,127,0.011249600350856781
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,255,0.015084800124168397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,511,0.016259199380874632
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,1023,0.0203247994184494
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,2,128,1,bfloat16,bfloat16,2047,0.020657600462436677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,2047,0.8311375617980957
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,3,0.010862399637699128
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,7,0.010840000212192535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,31,0.010883200168609618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,63,0.010956799983978272
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,127,0.011150400340557098
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,255,0.014985600113868713
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,511,0.01586720049381256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,1023,0.01998240053653717
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,15,0.010891199856996537
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,2047,0.020732800662517547
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,4,128,1,bfloat16,bfloat16,31,0.0182559996843338
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,1,0.010924799740314484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,3,0.01085119992494583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,7,0.011028800159692764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,31,0.010846400260925293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,127,0.011155200004577637
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,511,0.016225600242614747
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,255,0.015204800665378571
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,1023,0.02038400024175644
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,2047,0.02136639952659607
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,1,0.011430399864912033
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,3,0.011432000249624253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,7,0.011404799669981003
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,15,0.02985279858112335
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,31,0.011456000059843064
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,127,0.011659199744462967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,63,0.011521600186824799
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,255,0.015521599352359772
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,511,0.017076799273490907
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,1023,0.021305599808692934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,2047,0.021724799275398256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,1,0.0110384002327919
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,3,0.011072000116109848
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,7,0.011004800349473954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,15,0.011075200140476226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,31,0.01109279990196228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,63,0.011113599687814713
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,127,0.011299200356006622
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,255,0.015222400426864624
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,511,0.016356800496578217
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,1023,0.020552000403404234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,2,128,1,bfloat16,bfloat16,2047,0.02134400010108948
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,1,0.010868799686431885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,3,0.010876800119876861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,7,0.010857599973678588
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,15,0.010887999832630158
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,31,0.010976000130176545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,63,0.011020799726247787
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,127,0.011179199814796448
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,255,0.015084800124168397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,511,0.015971200168132783
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,4,128,1,bfloat16,bfloat16,1,0.010902400314807891
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,1023,0.020556800067424774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,4,128,1,bfloat16,bfloat16,2047,0.021540799736976625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,1,0.010993599891662598
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,3,0.010896000266075134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,7,0.010948800295591355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,15,0.011025600135326385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,31,0.010923200100660325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,63,0.011035200208425522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,127,0.011326400190591812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,255,0.015065599977970124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,511,0.01695999950170517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,1023,0.021158400177955627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,8,128,1,bfloat16,bfloat16,2047,0.025857600569725036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,1,0.011937599629163742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,3,0.01191679984331131
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,7,0.011931200325489045
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,31,0.012115199863910676
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,15,0.012041600048542022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,15,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,63,0.012113600224256515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,127,0.012375999987125397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,255,0.01613280028104782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,128,8,128,1,bfloat16,bfloat16,63,0.011187200248241425
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,511,0.025569599866867066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,1,0.011628799885511399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,2047,0.03879519999027252
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,3,0.012108799815177918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,15,0.011700800061225891
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,127,0.012107200175523757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,31,0.011817599833011627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,128,8,128,1,bfloat16,bfloat16,31,0.029985600709915163
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,255,0.016051200032234193
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,511,0.02829279899597168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,1023,0.036284801363945005
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,1,0.011664000153541566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,2047,0.056462401151657106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,7,0.011710400134325028
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,3,0.01165440008044243
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,31,0.011896000057458878
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,63,0.012001600116491318
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,127,0.01250080019235611
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,255,0.016487999260425566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,511,0.033083200454711914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,1023,0.05305280089378357
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,1,0.017502400279045104
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,2047,0.22401120662689208
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,3,0.01756799966096878
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,15,0.01751520037651062
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,31,0.01759359985589981
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,7,0.017584000527858735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,63,0.018012799322605133
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,127,0.0190080001950264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,255,0.026897600293159483
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,511,0.05580000281333923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,1023,0.22049920558929442
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,1,128,1,bfloat16,bfloat16,1023,0.03091840147972107
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,1,0.011409600079059602
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,8,128,1,bfloat16,bfloat16,2047,0.41671361923217776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,3,0.011416000127792359
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,15,0.01143999993801117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,63,0.012243200093507767
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,31,0.01156959980726242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,7,0.01146399974822998
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,63,0.012307199835777282
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,255,0.01555359959602356
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,1023,0.021355199813842773
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,511,0.0176704004406929
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,2047,0.02245279997587204
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,4,128,1,bfloat16,bfloat16,15,0.011838400363922119
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,3,0.011134400218725204
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,1,0.011505600064992905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,15,0.011145599931478501
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,63,0.011212799698114395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,31,0.012123200297355651
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,127,0.011396799981594086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,511,0.016568000614643096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,1023,0.021428799629211424
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,2047,0.02226399928331375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,1,0.010932800173759461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,3,0.010977599769830704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,7,0.01091040000319481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,15,0.01093439981341362
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,31,0.01093439981341362
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,127,0.011188799887895584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,255,0.015136000514030457
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,511,0.017284800112247468
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,1023,0.021404799818992615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,2047,0.0263808012008667
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,1,0.01096320003271103
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,3,0.01096320003271103
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,7,0.010900799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,15,0.010921599715948105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,128,2,128,1,bfloat16,bfloat16,7,0.011641599982976914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,63,0.01168000027537346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,128,1,128,1,bfloat16,bfloat16,15,0.01146719977259636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,1,128,1,bfloat16,bfloat16,127,0.011681599915027619
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,255,0.015387199819087982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,511,0.025115200877189638
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,1023,0.026368001103401185
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,1,0.014553600549697876
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,2047,0.0341839998960495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,7,0.01106560006737709
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,3,0.014553600549697876
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,7,0.014684799313545226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,31,0.014697599411010741
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,63,0.014998400211334228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,127,0.015225599706172942
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,255,0.019326399266719817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,511,0.036620798707008365
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,1,0.019809600710868836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,4,128,1,bfloat16,bfloat16,63,0.011072000116109848
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,1023,0.05798400044441223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,7,0.019662399590015412
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,3,0.01988160014152527
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,31,0.01977120041847229
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,63,0.020278400182723998
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,15,0.019900800287723543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,127,0.020824000239372253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,255,0.029097598791122437
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,511,0.060571199655532836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,1,0.030932798981666565
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,2,128,1,bfloat16,bfloat16,1023,0.2228624105453491
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,7,0.0311055988073349
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,31,0.010993599891662598
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,3,0.03081600069999695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,31,0.03127999901771546
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,15,0.030801600217819212
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,127,0.032969599962234496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,8,128,1,bfloat16,bfloat16,127,0.011724799871444702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,255,0.04886719882488251
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,511,0.2269968032836914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,1,0.05451200008392334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,3,0.05433120131492615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,31,0.05443840026855469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,15,0.05458880066871643
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,7,0.054232001304626465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,1023,0.42229437828063965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,63,0.05532640218734741
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,1,128,1,bfloat16,bfloat16,15,0.01465280055999756
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,127,0.05724800229072571
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,255,0.2151423931121826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,128,2,128,1,bfloat16,bfloat16,255,0.015619200468063355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,1,0.02325280010700226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,511,0.42095198631286623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,15,0.023196800053119658
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,7,0.02282560020685196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,3,0.0230880007147789
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,255,0.032574400305747986
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,1,0.03361760079860687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,127,0.024784000217914583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,63,0.02327360063791275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,3,0.033236798644065854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,31,0.03334720134735107
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,8,128,1,bfloat16,bfloat16,1023,0.8104640007019043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,7,0.03361279964447021
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,15,0.03371999859809875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,63,0.0338239997625351
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,127,0.03521600067615509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,3,0.05568959712982178
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,7,0.05574880242347717
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,1,0.056139200925827026
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,15,0.055776000022888184
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,31,0.055851197242736815
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,63,0.056883198022842404
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,127,0.05884000062942505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,4,128,1,bfloat16,bfloat16,255,0.22487521171569824
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,128,4,128,1,bfloat16,bfloat16,63,0.03144319951534271
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,3,0.09688799977302551
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,1,0.09719200134277343
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,7,0.09692800045013428
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,15,0.09727839827537536
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,31,0.09787359833717346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,127,0.22116799354553224
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,1,128,1,bfloat16,bfloat16,1,0.039215999841690066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,1,128,1,bfloat16,bfloat16,3,0.03877440094947815
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,255,0.41347517967224123
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,1,128,1,bfloat16,bfloat16,7,0.03945600092411041
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,1,128,1,bfloat16,bfloat16,15,0.0389631986618042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,1,128,1,bfloat16,bfloat16,127,0.04116640090942383
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,1,128,1,bfloat16,bfloat16,63,0.04015359878540039
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,1,128,1,bfloat16,bfloat16,31,0.039633598923683164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,1,128,1,bfloat16,bfloat16,31,0.023419199883937834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,2,128,1,bfloat16,bfloat16,1,0.06052799820899964
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,2,128,1,bfloat16,bfloat16,3,0.0606656014919281
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,2,128,1,bfloat16,bfloat16,7,0.06055520176887512
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,2,128,1,bfloat16,bfloat16,31,0.06023039817810059
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,2,128,1,bfloat16,bfloat16,15,0.060996800661087036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,2,128,1,bfloat16,bfloat16,63,0.0612064003944397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,2,128,1,bfloat16,bfloat16,127,0.06795359849929809
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,4,128,1,bfloat16,bfloat16,1,0.09978079795837402
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,4,128,1,bfloat16,bfloat16,3,0.10053919553756714
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,4,128,1,bfloat16,bfloat16,7,0.10044319629669189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,4,128,1,bfloat16,bfloat16,15,0.10038559436798096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,4,128,1,bfloat16,bfloat16,31,0.10086079835891723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,2,128,1,bfloat16,bfloat16,255,0.05222079753875732
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,4,128,1,bfloat16,bfloat16,63,0.10961920022964478
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,8,128,1,bfloat16,bfloat16,3,0.18280800580978393
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,4,128,1,bfloat16,bfloat16,127,0.24893760681152344
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,8,128,1,bfloat16,bfloat16,7,0.18330719470977783
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,8,128,1,bfloat16,bfloat16,15,0.18372319936752318
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,8,128,1,bfloat16,bfloat16,31,0.18358399868011474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,8,128,1,bfloat16,bfloat16,63,0.24744958877563478
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,1,0.011806400120258331
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,3,0.011430399864912033
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,128,8,128,1,bfloat16,bfloat16,63,0.09940000176429749
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,7,0.01156959980726242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,15,0.011793600022792816
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,127,0.011814399808645248
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,8,128,1,bfloat16,bfloat16,127,0.4384511947631836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,63,0.011598400026559829
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,255,0.015936000645160674
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,511,0.017870399355888366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,2047,0.02407200038433075
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,1,0.011508800089359283
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,15,0.011135999858379365
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,7,0.011187200248241425
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,31,0.011383999884128571
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,63,0.011187200248241425
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,127,0.011488000303506852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,255,0.015515199303627015
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,511,0.018086400628089905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,1023,0.022278399765491487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,1,0.010926400125026704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,2047,0.027276799082756042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,3,0.011032000184059143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,7,0.011052799969911575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,31,0.011057599633932113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,63,0.011048000305891037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,15,0.01117599979043007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,127,0.011343999952077865
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,255,0.015147200226783753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,511,0.021804800629615782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,1023,0.026734399795532226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,4,128,1,bfloat16,bfloat16,2047,0.03443840146064758
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,1,0.011243200302124024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,3,0.011068800091743469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,7,0.011025600135326385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,15,0.011307200044393539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,31,0.011112000048160552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,63,0.011206399649381638
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,127,0.011776000261306763
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,511,0.026009601354599
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,1023,0.03374240100383759
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,2047,0.053839999437332156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,128,8,128,1,bfloat16,bfloat16,1,0.18377599716186524
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,1,128,1,bfloat16,bfloat16,1,0.06486880183219909
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,1,128,1,bfloat16,bfloat16,3,0.06493600010871887
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,1,128,1,bfloat16,bfloat16,7,0.0653551995754242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,1,128,1,bfloat16,bfloat16,31,0.0656607985496521
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,1,128,1,bfloat16,bfloat16,15,0.06511359810829162
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,1,128,1,bfloat16,bfloat16,63,0.0818943977355957
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,2,128,1,bfloat16,bfloat16,1,0.10430079698562622
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,2,128,1,bfloat16,bfloat16,3,0.10454720258712769
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,2,128,1,bfloat16,bfloat16,7,0.10463199615478516
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,31,0.01156800016760826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,2,128,1,bfloat16,bfloat16,15,0.10443520545959473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,2,128,1,bfloat16,bfloat16,31,0.10554399490356445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,2,128,1,bfloat16,bfloat16,63,0.20132160186767578
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,4,128,1,bfloat16,bfloat16,3,0.18484480381011964
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,4,128,1,bfloat16,bfloat16,1,0.1841968059539795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,2,128,1,bfloat16,bfloat16,3,0.011132799834012986
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,1,128,1,bfloat16,bfloat16,1023,0.0230335995554924
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,4,128,1,bfloat16,bfloat16,15,0.1842847943305969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,4,128,1,bfloat16,bfloat16,7,0.18444960117340087
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,4,128,1,bfloat16,bfloat16,31,0.20425920486450194
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,4,128,1,bfloat16,bfloat16,63,0.29615519046783445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,8,128,1,bfloat16,bfloat16,1,0.35519680976867674
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,8,128,1,bfloat16,bfloat16,7,0.3552351951599121
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,8,128,1,bfloat16,bfloat16,3,0.35571041107177737
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,8,128,1,bfloat16,bfloat16,15,0.3582672119140625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,1,128,1,bfloat16,bfloat16,1,0.20984320640563964
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,8,128,1,bfloat16,bfloat16,31,0.3828160047531128
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,128,8,128,1,bfloat16,bfloat16,63,0.4820432186126709
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,1,128,1,bfloat16,bfloat16,3,0.2087631940841675
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,1,128,1,bfloat16,bfloat16,31,0.23537919521331788
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,1,128,1,bfloat16,bfloat16,15,0.21354238986968993
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,2,128,1,bfloat16,bfloat16,1,0.23980319499969482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,2,128,1,bfloat16,bfloat16,7,0.24624960422515868
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,2,128,1,bfloat16,bfloat16,15,0.2475264072418213
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,2,128,1,bfloat16,bfloat16,3,0.24078719615936278
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,2,128,1,bfloat16,bfloat16,31,0.2629760026931763
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,4,128,1,bfloat16,bfloat16,1,0.3909375905990601
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,4,128,1,bfloat16,bfloat16,3,0.3931663990020752
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,4,128,1,bfloat16,bfloat16,7,0.3968415975570679
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,4,128,1,bfloat16,bfloat16,15,0.39755840301513673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,4,128,1,bfloat16,bfloat16,31,0.4065375804901123
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,8,128,1,bfloat16,bfloat16,1,0.7356639862060547
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,8,128,1,bfloat16,bfloat16,3,0.7359295845031738
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,1,0.011574400216341018
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,8,128,1,bfloat16,bfloat16,7,0.7376031875610352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,7,0.011588799953460693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,3,0.011483199894428253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,8,128,1,bfloat16,bfloat16,15,0.739896011352539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,31,0.011635199934244157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,63,0.011692799627780914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,15,0.011591999977827071
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,8,128,1,bfloat16,bfloat16,31,0.7465360164642334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,511,0.020153599977493285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,1023,0.024377599358558655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,127,0.011929599940776825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,3,0.01117440015077591
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,7,0.011323200166225433
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,15,0.011220800131559372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,31,0.011318399757146835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,63,0.011267200112342834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,2047,0.029284799098968507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,127,0.011580800265073776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,255,0.015334400534629821
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,511,0.022728000581264497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,1023,0.027871999144554137
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,2047,0.035416001081466676
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,1,0.011088000237941742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,3,0.011161600053310395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,15,0.011116799712181092
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,31,0.011318399757146835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,63,0.011262399703264236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,128,1,128,1,bfloat16,bfloat16,7,0.21274399757385254
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,255,0.015468800067901611
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,128,8,128,1,bfloat16,bfloat16,255,0.015473599731922149
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,511,0.026855999231338502
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,1023,0.03426879942417145
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,2047,0.05428000092506409
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,3,0.011307200044393539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,1,0.0118367999792099
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,7,0.011347199976444244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,15,0.01130400002002716
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,63,0.011659199744462967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,127,0.012092799693346024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,31,0.011795199662446975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,255,0.016064000129699708
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,511,0.032232001423835754
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,1023,0.05275520086288452
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,1,0.012334399670362473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,8,128,1,bfloat16,bfloat16,2047,0.21978719234466554
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,7,0.012372799962759019
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,31,0.012399999797344208
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,15,0.012438400089740754
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,63,0.012491200119256973
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,127,0.012724800407886505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,255,0.016631999611854555
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,511,0.030273601412773132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,1023,0.03840959966182709
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,1,128,1,bfloat16,bfloat16,255,0.01576640009880066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,2047,0.058424001932144164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,1,0.012334399670362473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,3,0.012337599694728852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,15,0.012379200011491776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,2,128,1,bfloat16,bfloat16,1,0.01114879995584488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,7,0.012441600114107132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,31,0.012503999471664428
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,63,0.012611199915409089
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,127,0.012929600477218629
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,255,0.016945600509643555
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,511,0.03376800119876862
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,1,0.017985600233078002
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,1023,0.05379679799079895
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,3,0.017951999604701997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,2047,0.21833760738372804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,7,0.01794400066137314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,7,0.0110895998775959
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,63,0.01842080056667328
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,128,4,128,1,bfloat16,bfloat16,127,0.011588799953460693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,31,0.018063999712467194
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,255,0.027369600534439088
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,511,0.056944000720977786
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,1023,0.222760009765625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,1,0.029875200986862183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,7,0.029916799068450926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,3,0.029814401268959047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,15,0.029960000514984132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,2047,0.42491998672485354
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,31,0.029873600602149962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,63,0.0305184006690979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,255,0.04805760085582733
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,127,0.03184640109539032
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,511,0.21825120449066163
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,1,0.011265599727630615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,7,0.011244799941778183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,1023,0.41118879318237306
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,15,0.01128000020980835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,3,0.011135999858379365
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,63,0.011342400312423706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,127,0.011383999884128571
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,255,0.015332800149917603
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,511,0.01660960018634796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,31,0.011257600039243698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,1023,0.0208639994263649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,2047,0.021028800308704375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,1,0.01101439967751503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,3,0.01098880022764206
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,7,0.01091040000319481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,2047,0.8007984161376953
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,31,0.010956799983978272
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,63,0.010977599769830704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,255,0.015080000460147857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,511,0.0160303995013237
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,1023,0.020151999592781068
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,2047,0.02054080069065094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,1,0.010847999900579452
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,3,0.010836800187826156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,127,0.011104000359773636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,7,0.010868799686431885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,15,0.010815999656915664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,63,0.011062400043010711
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,127,0.011159999668598175
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,255,0.015052799880504609
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,511,0.015702399611473083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,15,0.017972800135612487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,1023,0.02001120001077652
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,2047,0.020776000618934632
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,1,0.01090560033917427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,7,0.01082720011472702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,15,0.010920000076293946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,127,0.019334399700164796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,31,0.010814400017261505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,63,0.01101280003786087
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,255,0.015081599354743958
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,127,0.011374399811029435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,511,0.015889599919319153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,1023,0.02051679939031601
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,1,0.011268799751996994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,2047,0.02157440036535263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,3,0.011088000237941742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,7,0.011267200112342834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,31,0.011188799887895584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,63,0.011336000263690948
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,127,0.011383999884128571
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,255,0.015388800203800202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,511,0.016686399281024934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,1023,0.020905600488185884
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,2047,0.021196800470352172
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,1,0.010971199721097946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,7,0.010943999886512757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,15,0.010929600149393082
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,31,0.010932800173759461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,63,0.01106399968266487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,127,0.011231999844312668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,3,0.01271039992570877
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,255,0.015115199983119965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,1023,0.02019840031862259
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,511,0.01644960045814514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,2047,0.021004800498485566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,3,0.010873600095510482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,7,0.010876800119876861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,15,0.010980799794197083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,15,0.010900799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,31,0.010966400057077408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,63,0.011078400164842605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,127,0.011164800077676774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,255,0.015095999836921692
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,511,0.016051200032234193
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,1023,0.02064319998025894
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,2047,0.021400000154972076
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,1,0.01096000000834465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,3,0.010903999954462052
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,31,0.010867200046777725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,15,0.010943999886512757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,31,0.0108815997838974
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,3,0.010887999832630158
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,63,0.01101439967751503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,127,0.011260800063610077
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,255,0.0150751993060112
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,511,0.016977599263191222
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,1023,0.02104319930076599
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,2047,0.025883200764656066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,1,0.011681599915027619
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,3,0.01167680025100708
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,7,0.011721599847078323
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,15,0.011742399632930755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,31,0.011763200163841248
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,15,0.011449600011110306
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,63,0.011878400295972823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,127,0.011999999731779098
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,255,0.016279999911785126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,511,0.024414399266242982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,1023,0.029364800453186034
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,1,0.011481600254774094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,3,0.010937599837779999
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,7,0.01148959994316101
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,15,0.011513599753379821
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,31,0.011615999788045884
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,63,0.011684799939393998
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,127,0.011931200325489045
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,255,0.01587519943714142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,511,0.027684798836708067
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,1023,0.035515201091766355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,1,0.011608000099658965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,2047,0.05519840121269226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,3,0.011691199988126755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,1,0.011564800143241882
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,15,0.011648000031709672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,31,0.012049599736928939
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,63,0.011953599750995636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,255,0.016441600024700166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,127,0.012759999930858612
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,511,0.033022400736808774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,1023,0.053544002771377566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,7,0.010956799983978272
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,2047,0.22187199592590331
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,3,0.017833599448204042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,1,0.01788959950208664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,15,0.017841599881649017
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,7,0.017601600289344786
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,31,0.017899200320243835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,127,0.018947200477123262
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,63,0.01825920045375824
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,255,0.027340799570083618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,511,0.05594239830970764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,1,0.01120320037007332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,1023,0.2195296049118042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,3,0.01117279976606369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,15,0.011220800131559372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,31,0.011235199868679047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,3,0.011470399796962738
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,2047,0.41586718559265134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,63,0.011315199732780456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,2047,0.03761439919471741
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,127,0.011606399714946748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,2047,0.02186879962682724
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,1023,0.021217599511146545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,255,0.015595200657844543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,1,0.011235199868679047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,7,0.011241599917411804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,3,0.012603199481964112
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,15,0.011147200316190719
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,31,0.011228799819946289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,127,0.011214400082826615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,255,0.0152319997549057
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,7,0.011622399836778641
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,1023,0.021171200275421142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,1,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,2047,0.0220223993062973
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,3,0.010945600271224976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,7,0.010947199910879135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,31,0.010964799672365189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,15,0.01101119965314865
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,63,0.01111999973654747
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,127,0.011228799819946289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,511,0.017257599532604216
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,255,0.015246400237083435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,1023,0.021244800090789794
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,2047,0.026080000400543212
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,3,0.010995200276374817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,1,0.011108800023794174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,7,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,15,0.010980799794197083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,31,0.011086399853229522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,127,0.011339200288057327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,255,0.015340800583362579
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,1023,0.026238399744033813
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,2047,0.03387520015239716
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,1,0.013947199285030364
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,3,0.014105600118637086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,7,0.013995200395584106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,7,0.011572799831628799
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,511,0.016731199622154237
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,15,0.014230400323867798
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,31,0.013928000628948212
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,63,0.014449599385261535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,127,0.014451199769973755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,255,0.01884160041809082
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,511,0.035606399178504944
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,1,0.019204799830913544
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,63,0.012113600224256515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,511,0.016648000478744505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,3,0.019171200692653656
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,1023,0.05634880065917969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,7,0.019670400023460387
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,63,0.019920000433921815
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,127,0.020737600326538087
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,255,0.02880159914493561
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,511,0.05934399962425232
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,63,0.011161600053310395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,511,0.021278400719165803
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,3,0.03083840012550354
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,7,0.030632001161575318
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,1,0.03097440004348755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,15,0.03075680136680603
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,1023,0.2199376106262207
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,31,0.03091999888420105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,63,0.031699201464653014
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,127,0.03294560015201568
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,255,0.049553599953651425
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,1,0.05427839756011963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,3,0.05441280007362366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,7,0.05437920093536377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,15,0.05452160239219665
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,31,0.05463520288467407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,1023,0.4180272102355957
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,63,0.055567997694015506
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,127,0.05755199790000916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,255,0.20799040794372559
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,1,0.021822400391101837
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,511,0.4165359973907471
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,7,0.021836799383163453
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,3,0.02189760059118271
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,31,0.019257600605487823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,31,0.021913599967956544
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,63,0.022385600209236144
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,15,0.021731199324131013
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,127,0.023585599660873414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,255,0.03130559921264649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,3,0.03259040117263794
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,15,0.032625600695610046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,31,0.03278880119323731
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,7,0.03296160101890564
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,63,0.03313120007514954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,127,0.034595200419425966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,1,0.05565599799156189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,3,0.05557439923286438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,7,0.05569760203361511
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,15,0.055587202310562134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,31,0.055979198217391966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,63,0.056380802392959596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,127,0.05878239870071411
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,511,0.22437601089477538
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,255,0.21761279106140136
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,1,0.09675040245056152
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,15,0.0969760000705719
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,7,0.09666240215301514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,3,0.0973039984703064
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,31,0.09731680154800415
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,63,0.0985696017742157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,127,0.21468639373779297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,1,128,1,bfloat16,bfloat16,1,0.03712159991264343
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,1,128,1,bfloat16,bfloat16,7,0.037319999933242795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,255,0.4067535877227783
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,1,128,1,bfloat16,bfloat16,15,0.03723520040512085
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,1,128,1,bfloat16,bfloat16,3,0.03790079951286316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,1,128,1,bfloat16,bfloat16,63,0.037878400087356566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,15,0.019438399374485014
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,1,128,1,bfloat16,bfloat16,31,0.03744480013847351
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,1,128,1,bfloat16,bfloat16,127,0.04015359878540039
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,2,128,1,bfloat16,bfloat16,1,0.05894560217857361
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,2,128,1,bfloat16,bfloat16,7,0.0591871976852417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,1023,0.8053456306457519
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,1,0.03269599974155426
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,2,128,1,bfloat16,bfloat16,3,0.05944960117340088
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,2,128,1,bfloat16,bfloat16,63,0.05965920090675354
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,2,128,1,bfloat16,bfloat16,15,0.05957279801368713
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,2,128,1,bfloat16,bfloat16,31,0.059271997213363646
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,4,128,1,bfloat16,bfloat16,3,0.09986240267753602
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,4,128,1,bfloat16,bfloat16,15,0.10007840394973755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,4,128,1,bfloat16,bfloat16,7,0.09990879893302917
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,4,128,1,bfloat16,bfloat16,1,0.09978399872779846
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,255,0.05138559937477112
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,4,128,1,bfloat16,bfloat16,31,0.10123039484024048
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,8,128,1,bfloat16,bfloat16,1,0.18292800188064576
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,4,128,1,bfloat16,bfloat16,127,0.23659200668334962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,8,128,1,bfloat16,bfloat16,3,0.18286240100860596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,8,128,1,bfloat16,bfloat16,7,0.18269920349121094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,8,128,1,bfloat16,bfloat16,15,0.18398239612579345
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,8,128,1,bfloat16,bfloat16,31,0.18440959453582764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,8,128,1,bfloat16,bfloat16,63,0.2371311902999878
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,3,0.01133119985461235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,1,0.011774399876594543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,31,0.01130559965968132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,7,0.011603199690580369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,15,0.011519999802112579
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,127,0.011547199636697768
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,63,0.011788800358772278
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,255,0.01582240015268326
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,1023,0.022011199593544008
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,511,0.017611199617385866
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,8,128,1,bfloat16,bfloat16,127,0.4219359874725342
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,2047,0.023612800240516662
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,3,0.011025600135326385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,1,0.01130239963531494
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,7,0.011166399717330933
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,63,0.011033599823713302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,31,0.011753600090742111
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,255,0.015248000621795654
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,127,0.011671999841928482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,15,0.011337599903345107
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,2047,0.026635199785232544
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,1,0.01125119999051094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,3,0.011142399907112122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,7,0.01101280003786087
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,15,0.011230400204658509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,31,0.011151999980211259
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,127,0.012593600153923034
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,511,0.02178719937801361
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,1023,0.026712000370025635
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,2047,0.03460640013217926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,2,128,1,bfloat16,bfloat16,127,0.062243199348449706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,1,0.01122559979557991
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,3,0.01101439967751503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,96,4,128,1,bfloat16,bfloat16,63,0.10361440181732177
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,7,0.011241599917411804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,15,0.011104000359773636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,127,0.0115167997777462
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,63,0.011395200341939925
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,31,0.01133119985461235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,255,0.015356799960136414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,511,0.02606239914894104
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,1023,0.03363519906997681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,1,128,1,bfloat16,bfloat16,1,0.06280959844589233
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,1,128,1,bfloat16,bfloat16,3,0.06286079883575439
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,1,128,1,bfloat16,bfloat16,7,0.06314240097999572
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,1,128,1,bfloat16,bfloat16,15,0.06307200193405152
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,1,128,1,bfloat16,bfloat16,31,0.06336479783058166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,1,128,1,bfloat16,bfloat16,63,0.0637440025806427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,2,128,1,bfloat16,bfloat16,1,0.10197440385818482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,511,0.017744000256061553
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,2,128,1,bfloat16,bfloat16,7,0.1021183967590332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,1023,0.022256000339984892
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,2,128,1,bfloat16,bfloat16,3,0.10204479694366456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,2,128,1,bfloat16,bfloat16,15,0.10195519924163818
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,63,0.011083199828863143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,2,128,1,bfloat16,bfloat16,31,0.10265599489212036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,255,0.016087999939918517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,4,128,1,bfloat16,bfloat16,3,0.18342239856719972
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,4,128,1,bfloat16,bfloat16,1,0.18361120223999022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,2,128,1,bfloat16,bfloat16,63,0.17780799865722657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,4,128,1,bfloat16,bfloat16,7,0.18453119993209838
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,4,128,1,bfloat16,bfloat16,15,0.18428640365600585
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,4,128,1,bfloat16,bfloat16,31,0.18793280124664308
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,4,128,1,bfloat16,bfloat16,63,0.2730544090270996
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,8,128,1,bfloat16,bfloat16,1,0.35449440479278566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,8,128,1,bfloat16,bfloat16,3,0.3548383951187134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,8,128,1,bfloat16,bfloat16,7,0.3552432060241699
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,8,128,1,bfloat16,bfloat16,15,0.3557967901229858
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,8,128,1,bfloat16,bfloat16,31,0.379366397857666
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,1,128,1,bfloat16,bfloat16,1,0.126256000995636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,1,128,1,bfloat16,bfloat16,3,0.13015199899673463
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,1,128,1,bfloat16,bfloat16,7,0.12484639883041382
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,1,128,1,bfloat16,bfloat16,15,0.13619840145111084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,1,128,1,bfloat16,bfloat16,31,0.1711151957511902
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,96,8,128,1,bfloat16,bfloat16,63,0.45957441329956056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,2,128,1,bfloat16,bfloat16,1,0.20906400680541992
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,2,128,1,bfloat16,bfloat16,7,0.21198720932006837
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,2,128,1,bfloat16,bfloat16,3,0.21197760105133057
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,2047,0.05368319749832153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,2,128,1,bfloat16,bfloat16,15,0.22172160148620607
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,2,128,1,bfloat16,bfloat16,31,0.23117599487304688
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,4,128,1,bfloat16,bfloat16,3,0.3816767930984497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,4,128,1,bfloat16,bfloat16,1,0.38020639419555663
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,4,128,1,bfloat16,bfloat16,7,0.38329761028289794
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,4,128,1,bfloat16,bfloat16,15,0.3863039970397949
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,4,128,1,bfloat16,bfloat16,31,0.3940000057220459
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,8,128,1,bfloat16,bfloat16,1,0.7292064189910888
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,1,0.011353600025177001
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,8,128,1,bfloat16,bfloat16,3,0.7308335781097413
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,7,0.011339200288057327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,15,0.011374399811029435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,8,128,1,bfloat16,bfloat16,7,0.7321199893951416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,127,0.011603199690580369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,3,0.011612799763679505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,8,128,1,bfloat16,bfloat16,15,0.7344751834869385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,31,0.011385600268840789
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,96,8,128,1,bfloat16,bfloat16,31,0.7437568187713623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,511,0.019211199879646302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,1,0.011104000359773636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,255,0.015481600165367126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,15,0.011051200330257416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,7,0.011073599755764007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,1023,0.023614400625228883
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,2047,0.028364801406860353
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,31,0.01111840009689331
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,63,0.011086399853229522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,255,0.015171200037002563
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,3,0.011043199896812439
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,127,0.011614400148391723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,511,0.02223999947309494
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,1,0.011020799726247787
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,2047,0.034969601035118106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,7,0.011131200194358825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,15,0.01109279990196228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,3,0.01130559965968132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,31,0.01106719970703125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,63,0.011267200112342834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,255,0.015751999616622925
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,511,0.02622399926185608
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,1023,0.03405439853668213
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,1,0.011292800307273865
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,3,0.011281599849462509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,7,0.011308799684047698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,2047,0.0546671986579895
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,15,0.011273600161075592
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,31,0.011289600282907486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,127,0.012019199877977371
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,63,0.011740799993276596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,511,0.03206399977207184
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,1023,0.052750402688980104
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,2047,0.22088799476623536
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,1,0.012144000083208085
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,7,0.012163200229406358
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,3,0.01199520006775856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,15,0.012191999703645706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,31,0.012110400199890136
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,63,0.012337599694728852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,127,0.012441600114107132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,255,0.016415999829769136
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,1023,0.036689600348472594
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,2047,0.056676799058914186
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,4095,0.227292799949646
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,1023,0.027209600806236266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,3,0.012135999649763108
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,7,0.012035199999809265
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,127,0.01154400035738945
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,15,0.012123200297355651
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,31,0.012179200351238251
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,63,0.012358400225639343
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,127,0.012873600423336028
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,255,0.01684959977865219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,511,0.033457601070404054
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,1023,0.05348960161209106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,2047,0.21760320663452148
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,1,0.018003199994564057
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,7,0.01801760047674179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,3,0.017873600125312805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,15,0.017955200374126436
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,4095,0.3953344106674194
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,31,0.01804800033569336
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,63,0.018449600040912627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,255,0.02729920148849487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,511,0.056435197591781616
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,127,0.019303999841213226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,1023,0.2204047918319702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,511,0.02874400019645691
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,2047,0.4177728176116943
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,1,0.029814401268959047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,63,0.011376000195741653
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,3,0.02972959876060486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,15,0.02974080145359039
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,31,0.029960000514984132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,7,0.02991360127925873
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,63,0.03039200007915497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,127,0.031836798787117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,255,0.0480783998966217
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,511,0.21672160625457765
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,1023,0.40930562019348143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,1,0.011044800281524658
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,7,0.011052799969911575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,1,0.012062399834394454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,15,0.0110384002327919
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,31,0.01098880022764206
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,63,0.01112319976091385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,3,0.010964799672365189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,2047,0.798967981338501
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,255,0.015182399749755859
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,511,0.01616320013999939
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,2047,0.020657600462436677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,4095,0.02128639966249466
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,1,0.010903999954462052
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,3,0.01088479980826378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,7,0.010878399759531022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,15,0.010911999642848969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,255,0.016044799983501435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,1023,0.02036159932613373
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,63,0.011006399989128113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,31,0.010985600203275681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,255,0.015043200552463531
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,1023,0.019896000623703003
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,511,0.015761600434780122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,2047,0.020270399749279022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,4095,0.021278400719165803
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,1,0.01085119992494583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,3,0.01085119992494583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,7,0.01090560033917427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,15,0.010864000022411346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,4095,1.5409903526306152
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,31,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,63,0.010969600081443787
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,127,0.011185599863529206
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,511,0.01571040004491806
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,1023,0.01985599994659424
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,2047,0.02067999988794327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,4095,0.021799999475479125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,1,0.010900799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,3,0.010886400192975997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,7,0.01090560033917427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,15,0.010838399827480315
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,31,0.010891199856996537
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,255,0.01499519944190979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,63,0.011147200316190719
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,127,0.011153600364923476
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,255,0.015020799636840821
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,1023,0.02046239972114563
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,2047,0.02106879949569702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,511,0.015568000078201295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,4095,0.025655999779701233
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,1,0.011044800281524658
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,3,0.01101600006222725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,31,0.011128000169992446
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,15,0.011036799848079681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,63,0.011155200004577637
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,7,0.010977599769830704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,127,0.011311999708414077
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,255,0.015169599652290344
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,511,0.016233600676059723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,2047,0.020787200331687926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,4095,0.7892608165740966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,1,0.010892800241708755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,4095,0.021726399660110474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,1023,0.020351999998092653
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,15,0.010867200046777725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,31,0.010913600027561188
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,63,0.010967999696731567
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,127,0.011179199814796448
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,255,0.015044799447059632
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,511,0.01584160029888153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,1023,0.020022399723529816
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,7,0.010864000022411346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,2047,0.020750400424003602
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,4095,0.021940800547599792
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,1,0.01085439994931221
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,7,0.010921599715948105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,3,0.01085439994931221
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,31,0.010894399881362916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,63,0.011055999994277954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,255,0.015086400508880615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,511,0.015958400070667268
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,1023,0.02048480063676834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,2047,0.021404799818992615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,127,0.011150400340557098
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,4095,0.025982400774955748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,1,0.010939200222492219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,7,0.010883200168609618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,15,0.010932800173759461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,31,0.010927999764680863
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,63,0.011088000237941742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,127,0.011236800253391266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,255,0.015166400372982025
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,511,0.016780799627304076
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,127,0.011262399703264236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,2047,0.025655999779701233
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,4095,0.033580800890922545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,1,0.011462400108575821
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,3,0.011521600186824799
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,7,0.011494400352239609
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,15,0.011563199758529662
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,31,0.011540800333023071
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,63,0.011606399714946748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,127,0.011740799993276596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,255,0.015673600137233734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,511,0.02322400063276291
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,1023,0.028043198585510253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,2047,0.03590719997882843
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,4095,0.055718398094177245
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,1,0.011433599889278412
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,7,0.011427199840545655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,3,0.01138719990849495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,15,0.011448000371456147
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,31,0.011507199704647064
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,63,0.01170559972524643
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,127,0.01194240003824234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,255,0.015820799767971037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,3,0.01091040000319481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,511,0.026915198564529418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,1023,0.034625598788261415
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,2047,0.05442720055580139
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,4095,0.22044000625610352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,3,0.01162559986114502
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,15,0.010864000022411346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,127,0.011150400340557098
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,7,0.011641599982976914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,31,0.012057600170373916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,127,0.012796799838542938
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,3,0.011713600158691407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,255,0.01686079949140549
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,511,0.03314880132675171
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,1023,0.020904000103473663
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,1023,0.053416001796722415
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,1,0.017824000120162962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,2047,0.2232032060623169
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,3,0.017846399545669557
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,4095,0.4053391933441162
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,15,0.017870399355888366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,7,0.017875200510025023
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,31,0.017574399709701538
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,63,0.018371200561523436
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,255,0.026812800765037538
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,127,0.019236800074577332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,511,0.05635679960250854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,1023,0.21752800941467285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,1,0.011115200072526931
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,2047,0.4161327838897705
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,3,0.011060799658298492
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,1,0.011640000343322753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,15,0.011135999858379365
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,31,0.011095999926328658
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,63,0.011184000223875046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,127,0.011315199732780456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,255,0.015136000514030457
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,7,0.01119839996099472
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,511,0.01624159961938858
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,15,0.011664000153541566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,1023,0.020454399287700653
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,4095,0.7842383861541748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,2047,0.021401600539684297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,3,0.010871999710798264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,31,0.011003199964761734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,63,0.011055999994277954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,7,0.011054400354623795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,127,0.011206399649381638
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,511,0.01610720008611679
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,1023,0.02083519995212555
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,2047,0.021580800414085388
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,4095,0.026078400015830994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,1,0.01096320003271103
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,3,0.010945600271224976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,7,0.010948800295591355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,15,0.01096159964799881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,31,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,63,0.01112319976091385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,127,0.011271999776363372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,255,0.015174399316310882
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,511,0.017047999799251555
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,1023,0.02106720060110092
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,2047,0.025950399041175843
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,4095,0.034179198741912845
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,1,0.010976000130176545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,3,0.010993599891662598
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,15,0.01098880022764206
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,31,0.010932800173759461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,63,0.0110895998775959
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,127,0.011243200302124024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,255,0.01512800008058548
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,4095,0.02255360037088394
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,1,0.011072000116109848
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,1023,0.026092800498008727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,2047,0.03389120101928711
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,4095,0.05379040241241455
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,1,0.01350879967212677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,255,0.015265600383281707
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,15,0.013299199938774108
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,3,0.013531200587749481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,31,0.013359999656677246
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,63,0.013804799318313599
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,255,0.017931200563907623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,127,0.013814400136470794
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,511,0.03535839915275574
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,1023,0.054996800422668454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,2047,0.2194063901901245
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,3,0.01870400011539459
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,15,0.018747200071811677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,7,0.018771199882030486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,31,0.018881599605083465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,63,0.019144000113010408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,127,0.02014240026473999
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,255,0.028016000986099243
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,511,0.05835999846458435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,7,0.011017599701881408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,1023,0.21647679805755615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,1,0.03049280047416687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,7,0.030609598755836485
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,2047,0.40828962326049806
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,3,0.03046720027923584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,15,0.0305184006690979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,63,0.011924800276756287
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,63,0.03111039996147156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,127,0.0326335996389389
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,31,0.030726400017738343
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,255,0.049332800507545474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,511,0.22159841060638427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,7,0.013550400733947754
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,1,0.05415199995040894
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,7,0.05421760082244873
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,3,0.05451359748840332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,15,0.054206401109695435
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,31,0.054929602146148684
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,63,0.05508959889411926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,2047,0.7988992214202881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,1,0.019020800292491914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,255,0.2036128044128418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,511,0.4122335910797119
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,1023,0.8010512351989746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,1,0.020982399582862854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,511,0.020929600298404693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,7,0.02098879963159561
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,3,0.020955200493335723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,15,0.011043199896812439
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,15,0.02089280039072037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,31,0.021238400042057036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,2047,1.6119184494018555
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,127,0.02213599979877472
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,255,0.03015359938144684
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,511,0.06217920184135437
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,3,0.032046398520469664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,7,0.03203200101852417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,1,0.03173919916152954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,31,0.03238239884376526
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,255,0.050670397281646726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,127,0.033843201398849485
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,127,0.05751680135726929
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,1,0.055313599109649655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,3,0.05567520260810852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,511,0.2238879919052124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,7,0.05532159805297852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,63,0.05623199939727783
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,15,0.05567520260810852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,31,0.055720001459121704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,127,0.058315199613571164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,255,0.2111232042312622
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,1,0.09669920206069946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,7,0.09681119918823242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,511,0.42080960273742674
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,3,0.09709920287132263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,15,0.09686880111694336
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,63,0.09903200268745423
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,63,0.021430400013923646
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,31,0.09736800193786621
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,127,0.20731840133666993
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,1,0.03596799969673157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,3,0.03582560122013092
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,7,0.03580479919910431
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,15,0.035883200168609616
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,255,0.3992016077041626
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,63,0.03642399907112122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,31,0.03604159951210022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,255,0.0542415976524353
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,511,0.7934016227722168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,1,128,1,bfloat16,bfloat16,127,0.03825919926166534
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,3,0.0582256019115448
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,15,0.05841599702835083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,31,0.05848640203475952
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,63,0.05957440137863159
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,7,0.05877280235290527
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,127,0.06126239895820618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,255,0.22196478843688966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,3,0.09928640127182006
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,1,0.09955840110778809
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,7,0.09954400062561035
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,31,0.10020320415496826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,63,0.102019202709198
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,127,0.22305119037628174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,1,0.18342560529708862
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,255,0.41503200531005857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,7,0.1837183952331543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,3,0.1828160047531128
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,1023,0.41506080627441405
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,31,0.18415039777755737
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,63,0.22617919445037843
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,63,0.03273119926452637
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,127,0.4071296215057373
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,1,0.011255999654531479
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,3,0.01173119992017746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,2,128,1,bfloat16,bfloat16,1,0.0580560028553009
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,15,0.011108800023794174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,31,0.011791999638080596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,7,0.01165440008044243
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,255,0.7903007984161377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,255,0.01558080017566681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,511,0.017059199512004852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,2047,0.022703999280929567
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,4095,0.02728320062160492
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,1,0.011776000261306763
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,7,0.01175519973039627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,4,128,1,bfloat16,bfloat16,15,0.09914079904556275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,15,0.011772800236940384
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,31,0.0110384002327919
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,127,0.011324799805879592
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,63,0.012963199615478515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,255,0.015185600519180298
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,511,0.017291200160980225
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,1023,0.02146400064229965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,2047,0.02614719867706299
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,4095,0.034376001358032225
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,1,0.01096000000834465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,3,0.010945600271224976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,7,0.010932800173759461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,64,8,128,1,bfloat16,bfloat16,15,0.1833184003829956
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,15,0.010950399935245514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,63,0.011112000048160552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,31,0.010995200276374817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,15,0.03208000063896179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,127,0.011271999776363372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,255,0.015217599272727967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,1023,0.026046401262283324
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,511,0.021686400473117828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,2047,0.03400000035762787
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,4095,0.0536191999912262
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,3,0.011043199896812439
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,127,0.01128000020980835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,63,0.011088000237941742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,15,0.011073599755764007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,31,0.011052799969911575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,7,0.011023999750614166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,127,0.011521600186824799
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,255,0.015414400398731232
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,511,0.02544800043106079
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,1023,0.03332160115242004
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,2047,0.05335519909858703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,1,128,1,bfloat16,bfloat16,1,0.06116799712181091
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,1023,0.02131039947271347
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,4095,0.22149920463562012
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,1,128,1,bfloat16,bfloat16,3,0.060734397172927855
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,1,128,1,bfloat16,bfloat16,7,0.06058400273323059
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,1,128,1,bfloat16,bfloat16,15,0.06090239882469177
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,3,0.011657600104808808
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,1,128,1,bfloat16,bfloat16,63,0.06098880171775818
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,1,128,1,bfloat16,bfloat16,31,0.0606112003326416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,1,128,1,bfloat16,bfloat16,127,0.07463039755821228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,2,128,1,bfloat16,bfloat16,1,0.1004207968711853
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,2,128,1,bfloat16,bfloat16,15,0.10026240348815918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,2,128,1,bfloat16,bfloat16,7,0.10040960311889649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,2,128,1,bfloat16,bfloat16,3,0.10071519613265992
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,2,128,1,bfloat16,bfloat16,31,0.10100959539413452
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,2,128,1,bfloat16,bfloat16,63,0.11021280288696289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,4,128,1,bfloat16,bfloat16,1,0.1832383990287781
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,2,128,1,bfloat16,bfloat16,127,0.24440159797668456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,4,128,1,bfloat16,bfloat16,7,0.1838655948638916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,4,128,1,bfloat16,bfloat16,15,0.183460795879364
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,4,128,1,bfloat16,bfloat16,31,0.18392159938812255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,4,128,1,bfloat16,bfloat16,63,0.2506335973739624
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,8,128,1,bfloat16,bfloat16,1,0.35524160861968995
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,4,128,1,bfloat16,bfloat16,127,0.4327888011932373
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,8,128,1,bfloat16,bfloat16,3,0.3546015977859497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,8,128,1,bfloat16,bfloat16,7,0.35427520275115965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,8,128,1,bfloat16,bfloat16,15,0.3552479982376099
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,8,128,1,bfloat16,bfloat16,31,0.37259199619293215
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,1,128,1,bfloat16,bfloat16,1,0.10499199628829955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,1,0.011390399932861329
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,1,128,1,bfloat16,bfloat16,3,0.10444480180740356
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,1,128,1,bfloat16,bfloat16,7,0.10513759851455688
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,63,0.0114656001329422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,1,128,1,bfloat16,bfloat16,31,0.10623999834060668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,8,128,1,bfloat16,bfloat16,127,0.8106224060058593
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,1,128,1,bfloat16,bfloat16,15,0.10497759580612183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,1,128,1,bfloat16,bfloat16,63,0.20086560249328614
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,2,128,1,bfloat16,bfloat16,1,0.1841599941253662
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,2,128,1,bfloat16,bfloat16,3,0.18398239612579345
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,2,128,1,bfloat16,bfloat16,31,0.20313920974731445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,2,128,1,bfloat16,bfloat16,15,0.18462879657745362
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,2,128,1,bfloat16,bfloat16,7,0.18435519933700562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,4,128,1,bfloat16,bfloat16,1,0.3545583963394165
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,4,128,1,bfloat16,bfloat16,3,0.35599360466003416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,2,128,1,bfloat16,bfloat16,63,0.2967312097549438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,4,128,1,bfloat16,bfloat16,7,0.35594561100006106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,4,128,1,bfloat16,bfloat16,15,0.36162400245666504
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,4,128,1,bfloat16,bfloat16,3,0.18391519784927368
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,4,128,1,bfloat16,bfloat16,31,0.3858544111251831
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,4,128,1,bfloat16,bfloat16,63,0.49152798652648927
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,8,128,1,bfloat16,bfloat16,1,0.6958255767822266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,8,128,1,bfloat16,bfloat16,7,0.7001904010772705
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,8,128,1,bfloat16,bfloat16,3,0.6967535972595215
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,1,0.011196800321340562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,3,0.011166399717330933
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,7,0.011275199800729751
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,8,128,1,bfloat16,bfloat16,15,0.7273712158203125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,15,0.011268799751996994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,31,0.011191999912261963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,8,128,1,bfloat16,bfloat16,31,0.7354623794555664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,63,0.011407999694347382
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,511,0.018116800487041472
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,255,0.015404799580574035
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,64,8,128,1,bfloat16,bfloat16,63,0.43827037811279296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,64,8,128,1,bfloat16,bfloat16,63,0.8629504203796386
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,4095,0.035385599732398985
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,1023,0.022060799598693847
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,15,0.010999999940395355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,31,0.01101280003786087
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,3,0.010945600271224976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,63,0.011139199882745743
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,127,0.01127680018544197
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,7,0.010955200344324113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,255,0.015238399803638458
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,511,0.021704000234603883
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,2047,0.03439359962940216
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,4095,0.05435680150985718
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,1023,0.02674719989299774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,1,0.011048000305891037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,3,0.011057599633932113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,7,0.011126399785280228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,31,0.011051200330257416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,63,0.01125440001487732
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,127,0.011503999680280685
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,255,0.015460799634456634
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,511,0.02588160037994385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,1023,0.03358719944953918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,2047,0.05374559760093689
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,1,0.011270400136709213
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,4095,0.22559359073638915
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,3,0.011318399757146835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,15,0.011336000263690948
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,31,0.011393599957227708
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,7,0.011244799941778183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,63,0.01156959980726242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,127,0.012064000219106674
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,255,0.016033600270748138
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,511,0.03212479948997497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,1023,0.0520687997341156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,1,0.011812800168991089
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,2047,0.2199471950531006
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,3,0.01173119992017746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,15,0.011825600266456604
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,7,0.011825600266456604
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,31,0.011811199784278869
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,4095,0.40333919525146483
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,127,0.012507200241088867
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,63,0.011961600184440613
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,127,0.012251199781894683
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,1023,0.035148799419403076
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,255,0.01611039936542511
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,511,0.02773439884185791
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,2047,0.055099201202392575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,1,0.012375999987125397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,3,0.012006399780511856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,15,0.01202239990234375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,4095,0.22153921127319337
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,7,0.012383999675512314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,8191,0.4033599853515625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,31,0.012088000029325485
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,15,0.011059200018644333
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,127,0.013143999874591828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,63,0.01231359988451004
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,511,0.0332751989364624
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,255,0.01677599996328354
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,1023,0.0529263973236084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,1,0.017833599448204042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,2047,0.21527199745178222
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,7,0.0178384006023407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,3,0.01791200041770935
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,4095,0.3944384098052979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,31,0.017931200563907623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,63,0.018297599256038667
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,255,0.02727360129356384
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,511,0.05599679946899414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,127,0.019284799695014954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,8191,0.7519231796264648
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,1023,0.2199471950531006
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,2047,0.4159232139587402
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,1,0.029678401350975037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,7,0.029713600873947144
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,15,0.029737600684165956
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,3,0.02969599962234497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,31,0.029864001274108886
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,2047,0.02722240090370178
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,1,0.011308799684047698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,63,0.03038559854030609
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,127,0.0317903995513916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,255,0.04843519926071167
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,511,0.21503520011901855
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,1023,0.4071040153503418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,8191,1.5636079788208008
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,1,0.010889600217342376
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,2047,0.7973104000091553
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,7,0.010886400192975997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,31,0.010931199789047242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,63,0.01096000000834465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,3,0.010793600231409073
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,127,0.011161600053310395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,15,0.01085280030965805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,511,0.015904000401496886
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,255,0.015011200308799743
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,1023,0.02003040015697479
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,8191,0.022171199321746826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,4095,0.020744000375270844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,1,0.010742399841547012
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,15,0.01828960031270981
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,7,0.0107744000852108
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,4095,1.5359456062316894
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,3,0.01125440001487732
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,31,0.010795199871063232
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,15,0.01157120019197464
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,127,0.011075200140476226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,255,0.015379199385643005
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,63,0.011129599809646607
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,2047,0.02048639953136444
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,8191,0.023204800486564637
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,4095,0.021300800144672394
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,1,0.011500799655914306
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,4095,0.7874864101409912
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,3,0.011033599823713302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,7,0.011503999680280685
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,31,0.011105599999427795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,63,0.012649600207805634
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,15,0.010819199681282043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,127,0.011416000127792359
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,255,0.015299199521541596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,8191,3.009881591796875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,1023,0.019844800233840942
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,511,0.015371200442314149
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,2047,0.02072319984436035
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,1,0.010824000090360641
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,8191,0.026475200057029726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,7,0.010791999846696853
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,3,0.011800000071525573
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,15,0.01090560033917427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,31,0.01085119992494583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,127,0.011184000223875046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,63,0.011540800333023071
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,255,0.015091200172901154
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,511,0.0156016007065773
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,2047,0.021048000454902648
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,4095,0.025433599948883057
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,8191,0.034139201045036316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,1,0.010903999954462052
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,3,0.01085280030965805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,7,0.010864000022411346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,2047,0.020207999646663664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,31,0.010887999832630158
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,63,0.010976000130176545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,127,0.011247999966144562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,255,0.015056000649929046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,511,0.015803200006484986
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,1023,0.019809600710868836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,2047,0.020267200469970704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,4095,0.02117439955472946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,8191,0.022987200319766997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,1,0.010836800187826156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,3,0.010883200168609618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,7,0.010859200358390808
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,511,0.01573760062456131
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,31,0.010911999642848969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,63,0.011025600135326385
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,127,0.01117599979043007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,255,0.015059199929237366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,511,0.01565600037574768
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,1023,0.019959999620914458
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,2047,0.02069920003414154
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,4095,0.021851199865341186
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,8191,0.026399999856948853
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,3,0.010860799998044967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,7,0.010849600285291671
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,15,0.010871999710798264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,31,0.010900799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,63,0.011046399921178817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,127,0.011217600107192994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,255,0.015080000460147857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,511,0.015732799470424653
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,1023,0.02027200013399124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,2047,0.021038399636745454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,4095,0.02561120092868805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,4095,0.021531200408935545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,8191,0.034134399890899655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,1,0.0108255997300148
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,3,0.010920000076293946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,7,0.010887999832630158
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,1023,0.020667199790477753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,15,0.01091040000319481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,31,0.010899200290441512
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,127,0.011230400204658509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,63,0.011547199636697768
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,511,0.016598400473594666
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,15,0.01082720011472702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,1023,0.021147200465202333
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,2047,0.025526401400566102
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,4095,0.033934399485588074
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,1,0.011289600282907486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,7,0.011307200044393539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,8191,0.053814399242401126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,3,0.011499200016260147
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,15,0.011340799927711486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,63,0.011619199812412263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,1023,0.019985599815845488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,15,0.010843200236558914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,255,0.015700800716876982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,511,0.0220768004655838
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,1023,0.027000001072883605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,2047,0.0351967990398407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,4095,0.05450080037117004
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,1,0.01082720011472702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,1,0.011604800075292587
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,3,0.01138719990849495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,8191,0.21852641105651854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,15,0.011449600011110306
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,7,0.01156800016760826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,63,0.011591999977827071
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,127,0.012059199810028075
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,31,0.0114656001329422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,255,0.015863999724388123
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,511,0.026659199595451356
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,1023,0.03414239883422852
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,2047,0.05420799851417542
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,1,0.01159999966621399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,3,0.011736000329256058
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,4095,0.21902239322662354
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,7,0.011641599982976914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,31,0.01173280030488968
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,8191,0.39389920234680176
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,63,0.011929599940776825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,15,0.011796800047159195
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,255,0.01646080017089844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,127,0.012392000108957291
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,511,0.03269760012626648
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,1023,0.05291680097579956
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,255,0.015062400698661804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,2047,0.22109439373016357
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,3,0.01752000004053116
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,15,0.017524799704551695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,4095,0.40526399612426756
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,31,0.017550399899482726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,127,0.01886879950761795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,127,0.011643200367689132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,63,0.01797119975090027
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,31,0.011694400012493134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,8191,0.7715407848358155
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,511,0.0552079975605011
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,1023,0.21626560688018798
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,1,0.010844799876213073
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,2047,0.41657438278198244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,3,0.010873600095510482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,15,0.010820800065994262
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,31,0.010844799876213073
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,4095,0.7849760055541992
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,7,0.011580800265073776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,255,0.015457600355148315
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,127,0.011215999722480774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,511,0.016201600432395935
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,1023,0.02008800059556961
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,2047,0.02109919935464859
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,4095,0.021982400119304656
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,8191,1.5303248405456542
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,8191,0.027209600806236266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,1,0.010846400260925293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,15,0.010820800065994262
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,7,0.011577600240707397
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,3,0.010913600027561188
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,63,0.013625599443912506
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,127,0.011238399893045425
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,255,0.015467199683189391
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,1,0.017800000309944154
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,511,0.01613280028104782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,1023,0.02081120014190674
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,2047,0.021747200191020964
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,7,0.017526400089263917
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,3,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,4095,0.02608799934387207
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,1,0.01117599979043007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,255,0.026841598749160766
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,31,0.01120320037007332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,15,0.010899200290441512
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,255,0.01509920060634613
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,127,0.0114656001329422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,511,0.016764800250530242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,2047,0.02574560046195984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,1023,0.021040000021457672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,4095,0.03370879888534546
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,1,0.010932800173759461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,3,0.010924799740314484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,7,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,63,0.010918399691581726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,15,0.01096159964799881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,31,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,63,0.011112000048160552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,127,0.011257600039243698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,255,0.015158399939537048
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,511,0.020948800444602966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,1023,0.02566559910774231
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,2047,0.033508801460266115
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,4095,0.05329599976539612
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,1,0.012612800300121307
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,8191,0.21985280513763428
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,3,0.0126351997256279
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,31,0.010875199735164643
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,15,0.012608000636100769
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,7,0.012577599287033081
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,63,0.012921600043773651
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,31,0.012763200700283051
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,127,0.01345600038766861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,255,0.017401599884033205
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,511,0.034441599249839784
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,1023,0.05474240183830261
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,1,0.018615999817848207
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,2047,0.21617441177368163
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,3,0.018545599281787874
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,8191,0.03454560041427612
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,7,0.010883200168609618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,15,0.01855839937925339
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,7,0.018555200099945067
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,4095,0.39911201000213625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,63,0.011112000048160552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,31,0.018838399648666383
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,127,0.019968000054359437
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,63,0.01934719979763031
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,255,0.027897599339485168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,1,0.030686399340629576
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,511,0.05782560110092163
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,3,0.0304639995098114
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,1023,0.21464641094207765
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,31,0.030532801151275636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,8191,0.053748798370361325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,2047,0.4059887886047363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,127,0.03245919942855835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,63,0.031384000182151796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,4095,0.7630176067352294
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,511,0.21791999340057372
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,1023,0.41028318405151365
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,1,0.05435839891433716
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,3,0.053995198011398314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,7,0.054548799991607666
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,15,0.0541055977344513
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,31,0.05474720001220703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,63,0.05507519841194153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,127,0.05744479894638062
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,255,0.19993120431900024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,511,0.40639519691467285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,4095,1.5641615867614747
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,1023,0.7938047885894776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,15,0.030427199602127076
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,1,0.01971199959516525
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,3,0.019652800261974336
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,15,0.01977919936180115
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,7,0.019679999351501463
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,31,0.01993599981069565
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,63,0.02018879950046539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,127,0.02115039974451065
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,255,0.029142400622367857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,511,0.060139197111129764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,1023,0.21894559860229493
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,1,0.03151200115680695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,2047,1.6089775085449218
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,7,0.03159680068492889
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,3,0.03148959875106812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,2047,0.7964079856872559
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,31,0.03172479867935181
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,63,0.03214559853076935
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,15,0.03156639933586121
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,255,0.05000960230827332
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,127,0.033876800537109376
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,511,0.21648800373077393
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,1,0.05528479814529419
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,15,0.055232000350952146
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,7,0.05518239736557007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,31,0.05555679798126221
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,63,0.05595200061798096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,1023,0.4068160057067871
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,127,0.05838879942893982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,255,0.20431358814239503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,4095,3.1958703994750977
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,511,0.4123727798461914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,1,0.09653599858283997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,7,0.030768001079559328
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,15,0.09675840139389039
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,7,0.09671679735183716
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,255,0.04907839894294739
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,63,0.09826080203056335
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,1023,0.7992623805999756
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,3,0.0964896023273468
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,31,0.09724159836769104
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,127,0.19990559816360473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,1,0.03438720107078552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,3,0.0346560001373291
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,7,0.03440960049629212
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,15,0.034292799234390256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,31,0.03495199978351593
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,255,0.3908112049102783
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,63,0.035206401348114015
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,1,0.05777599811553955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,511,0.7808432102203369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,127,0.036392000317573545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,255,0.053320002555847165
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,3,0.05804160237312317
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,7,0.0578544020652771
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,1,128,1,bfloat16,bfloat16,511,0.2278592109680176
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,31,0.05784479975700378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,15,0.05761600136756897
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,127,0.06133279800415039
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,255,0.2087167978286743
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,3,0.055272001028060916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,7,0.09977759718894959
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,1,0.09917759895324707
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,31,0.09987999796867371
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,15,0.09928799867630005
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,1023,1.551419162750244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,511,0.41274399757385255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,63,0.10183520317077636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,127,0.20930240154266358
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,3,0.18312959671020507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,255,0.4020319938659668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,1,0.1823024034500122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,7,0.18280800580978393
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,31,0.18345439434051514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,63,0.21702399253845214
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,511,0.8353967666625977
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,127,0.3932287931442261
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,1,0.010873600095510482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,7,0.010873600095510482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,3,0.012831999361515046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,255,0.7728047847747803
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,15,0.0108815997838974
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,127,0.011177600175142289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,31,0.011596799641847611
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,255,0.015140800178050995
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,511,0.016318400204181672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,1023,0.020838400721549986
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,63,0.01106719970703125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,2047,0.021803200244903564
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,2,128,1,bfloat16,bfloat16,63,0.05917760133743286
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,1,0.011131200194358825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,8191,0.034622400999069214
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,511,1.5340160369873046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,7,0.011150400340557098
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,4,128,1,bfloat16,bfloat16,3,0.09910560250282288
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,63,0.011236800253391266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,127,0.011193600296974183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,255,0.015409600734710694
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,511,0.017067199945449828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,2047,0.025900799036026
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,31,0.010865599662065507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,4095,0.03409920036792755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,1,0.010969600081443787
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,3,0.010940799862146378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,8191,0.0542352020740509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,32,8,128,1,bfloat16,bfloat16,15,0.18312000036239623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,7,0.010947199910879135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,63,0.011033599823713302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,31,0.011019200086593628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,127,0.011264000087976456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,15,0.010892800241708755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,255,0.015177600085735321
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,511,0.021009600162506102
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,1023,0.025644800066947936
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,4095,0.0529807984828949
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,1,0.01101439967751503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,8191,0.22308480739593506
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,3,0.011030399799346923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,15,0.011088000237941742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,31,0.011083199828863143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,7,0.010974399745464325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,63,0.011208000034093857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,127,0.011478400230407715
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,255,0.015435199439525604
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,511,0.02534399926662445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,1023,0.03306719958782196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,2047,0.05347359776496887
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,4095,0.02616479992866516
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,4095,0.22124640941619872
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,1,0.058499199151992795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,3,0.012188799679279327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,7,0.058671998977661136
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,15,0.01101280003786087
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,3,0.058659201860427855
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,8191,0.40464000701904296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,31,0.05866879820823669
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,15,0.05886080265045166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,63,0.05956000089645386
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,127,0.06164960265159607
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,1,0.09959359765052796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,1023,0.0213919997215271
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,1,128,1,bfloat16,bfloat16,255,0.2242959976196289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,3,0.09986720085144044
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,15,0.09945120215415955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,31,0.10023679733276367
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,63,0.10206559896469117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,127,0.2184528112411499
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,1,0.18308160305023194
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,3,0.18356640338897706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,255,0.4058207988739014
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,7,0.18315680027008058
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,15,0.18284959793090821
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,31,0.1837712049484253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,63,0.23030400276184082
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,2047,0.033555200695991515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,1,0.3532975912094116
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,3,0.35366880893707275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,127,0.4049392223358154
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,7,0.3551088094711304
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,15,0.3557984113693237
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,31,0.3585824012756348
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,63,0.4219791889190674
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,4,128,1,bfloat16,bfloat16,255,0.7913519859313964
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,1,128,1,bfloat16,bfloat16,3,0.10100799798965454
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,1,128,1,bfloat16,bfloat16,7,0.10079519748687744
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,127,0.7760879993438721
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,1,128,1,bfloat16,bfloat16,15,0.10056320428848267
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,1,128,1,bfloat16,bfloat16,63,0.11175999641418458
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,1,128,1,bfloat16,bfloat16,31,0.10178719758987427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,2,128,1,bfloat16,bfloat16,1,0.18298239707946778
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,2,128,1,bfloat16,bfloat16,3,0.18311519622802735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,1,128,1,bfloat16,bfloat16,127,0.2552175998687744
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,2,128,1,bfloat16,bfloat16,15,0.18293440341949463
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,2,128,1,bfloat16,bfloat16,7,0.18428319692611694
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,2,128,1,bfloat16,bfloat16,31,0.18383359909057617
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,2,128,1,bfloat16,bfloat16,63,0.2520607948303223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,2,128,1,bfloat16,bfloat16,7,0.09956319928169251
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,32,8,128,1,bfloat16,bfloat16,255,1.5390255928039551
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,4,128,1,bfloat16,bfloat16,1,0.3560960054397583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,2,128,1,bfloat16,bfloat16,127,0.42445921897888184
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,4,128,1,bfloat16,bfloat16,3,0.3542288064956665
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,4,128,1,bfloat16,bfloat16,7,0.35454559326171875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,4,128,1,bfloat16,bfloat16,15,0.35530240535736085
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,4,128,1,bfloat16,bfloat16,31,0.37555680274963377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,4,128,1,bfloat16,bfloat16,63,0.45129919052124023
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,8,128,1,bfloat16,bfloat16,1,0.6940000057220459
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,8,128,1,bfloat16,bfloat16,3,0.6937007904052734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,8,128,1,bfloat16,bfloat16,7,0.6970655918121338
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,4,128,1,bfloat16,bfloat16,127,0.7963647842407227
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,1,128,1,bfloat16,bfloat16,1,0.10056480169296264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,8,128,1,bfloat16,bfloat16,15,0.695740795135498
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,8,128,1,bfloat16,bfloat16,31,0.7299600124359131
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,1,0.011526399850845337
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,7,0.010911999642848969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,3,0.010915199667215348
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,8,128,1,bfloat16,bfloat16,63,0.8268912315368653
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,63,0.011505600064992905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,15,0.010955200344324113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,127,0.011260800063610077
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,31,0.010974399745464325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,255,0.01512800008058548
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,1023,0.021833600103855134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,2047,0.026361599564552307
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,4095,0.03454560041427612
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,1,0.010943999886512757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,511,0.018545599281787874
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,8191,0.054153597354888915
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,7,0.010983999818563461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,31,0.010979200154542923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,255,0.015167999267578124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,127,0.011275199800729751
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,511,0.021294400095939636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,1023,0.026174399256706237
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,32,8,128,1,bfloat16,bfloat16,127,1.5307007789611817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,4095,0.05379040241241455
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,3,0.011022400110960007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,8191,0.2160576105117798
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,7,0.011044800281524658
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,31,0.011027199774980545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,1,0.011068800091743469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,15,0.011121600121259689
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,63,0.011214400082826615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,255,0.015433600544929505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,511,0.025545600056648254
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,127,0.011502400040626526
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,1023,0.03325279951095581
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,2047,0.0534608006477356
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,1,0.011209599673748016
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,3,0.011217600107192994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,4095,0.23981120586395263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,7,0.011283200234174728
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,8191,0.40452799797058103
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,31,0.011315199732780456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,127,0.012009599804878235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,15,0.01130559965968132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,255,0.01603520065546036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,63,0.011563199758529662
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,511,0.03203519880771637
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,1023,0.052223998308181765
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,2047,0.21857600212097167
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,1,0.0118367999792099
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,3,0.011740799993276596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,7,0.01180799975991249
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,15,0.011803200095891952
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,3,0.020771199464797975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,15,0.010948800295591355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,31,0.011843200027942657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,8191,0.7733151912689209
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,63,0.01215839982032776
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,63,0.011843200027942657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,255,0.016172799468040466
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,127,0.012532800436019897
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,2047,0.03400799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,1023,0.0348800003528595
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,511,0.026878398656845093
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,4095,0.21779520511627198
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,1,0.012211199849843979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,3,0.01197120025753975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,8191,0.4004655838012695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,15,0.012267199903726577
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,31,0.012036799639463424
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,7,0.012049599736928939
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,255,0.016739200055599212
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,511,0.03296479880809784
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,63,0.012620800733566284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,16383,0.7687583923339844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,1023,0.05287839770317078
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,2047,0.21435999870300293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,4095,0.40287041664123535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,1,0.017876799404621124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,8191,0.7517151832580566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,3,0.017972800135612487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,7,0.017880000174045563
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,31,0.018059200048446654
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,63,0.01828159987926483
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,4095,0.4022575855255127
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,127,0.019308799505233766
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,255,0.027275198698043825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,15,0.017926399409770966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,511,0.0557856023311615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,2047,0.41515841484069826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,16383,1.4814767837524414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,2047,0.054841601848602296
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,4095,0.7846799850463867
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,3,0.030118399858474733
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,1,0.02975359857082367
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,15,0.030249598622322082
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,7,0.02980000078678131
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,127,0.012964800000190735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,31,0.029979199171066284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,63,0.030823999643325807
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,127,0.03172639906406403
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,255,0.048895999789237976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,8191,1.5310447692871094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,511,0.22931039333343506
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,1023,0.4059904098510742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,2047,0.7947391986846923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,1,0.01111840009689331
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,7,0.011128000169992446
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,3,0.010780800133943558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,15,0.011137600243091583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,31,0.010807999968528747
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,63,0.011228799819946289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,127,0.011135999858379365
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,1023,0.21798560619354249
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,255,0.015372799336910247
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,4095,1.5372223854064941
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,2047,0.020452800393104553
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,1023,0.019726400077342988
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,16383,3.0044607162475585
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,16383,0.024928000569343568
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,8191,0.022235199809074402
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,3,0.010760000348091126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,4095,0.020688000321388244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,7,0.011023999750614166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,15,0.010806400328874588
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,127,0.0110384002327919
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,63,0.011142399907112122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,31,0.011019200086593628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,255,0.014947199821472168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,511,0.015670399367809295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,2047,0.01969279944896698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,1023,0.01960960030555725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,8191,0.022555199265480042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,4095,0.021076799929142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,1,0.010763200372457505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,3,0.011033599823713302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,7,0.010777600109577179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,15,0.011032000184059143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,31,0.010793600231409073
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,16383,0.02779200077056885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,127,0.0110944002866745
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,63,0.011156799644231797
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,255,0.015052799880504609
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,511,0.015255999565124512
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,2047,0.0202224001288414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,1023,0.019438399374485014
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,8191,3.0986576080322266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,8191,0.026081600785255434
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,16383,0.03504959940910339
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,3,0.010918399691581726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,7,0.010878399759531022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,15,0.010972800105810166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,31,0.010927999764680863
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,63,0.011068800091743469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,127,0.011243200302124024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,1,0.010892800241708755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,255,0.01510239988565445
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,1023,0.020473599433898926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,4095,0.02547680139541626
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,8191,0.03402880132198334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,1,0.010840000212192535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,16383,0.05355679988861084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,3,0.010828799754381179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,15,0.010819199681282043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,511,0.015603199601173401
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,31,0.010824000090360641
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,63,0.010942400246858597
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,127,0.011078400164842605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,255,0.015056000649929046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,511,0.015612800419330598
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,1023,0.019679999351501463
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,2047,0.0200655996799469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,4095,0.021180799603462218
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,8191,0.02295680046081543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,16383,0.028092798590660096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,1,0.010891199856996537
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,1,0.010864000022411346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,3,0.01085280030965805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,7,0.010849600285291671
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,15,0.0108815997838974
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,31,0.010947199910879135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,127,0.011147200316190719
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,255,0.014990399777889251
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,511,0.015492799878120422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,16383,5.952497482299805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,1023,0.01964319944381714
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,4095,0.02160799950361252
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,8191,0.026183998584747313
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,4095,0.021422399580478667
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,16383,0.034959998726844785
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,2047,0.02040800005197525
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,3,0.010883200168609618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,15,0.010897599905729295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,7,0.010833600163459777
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,31,0.010897599905729295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,127,0.011164800077676774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,63,0.010950399935245514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,255,0.015094399452209473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,1023,0.020270399749279022
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,2047,0.021076799929142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,511,0.015556800365447997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,2047,0.021078400313854218
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,4095,0.025579199194908142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,1,0.010926400125026704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,16383,0.05355520248413086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,7,0.010715200006961823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,3,0.010923200100660325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,15,0.010924799740314484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,31,0.010942400246858597
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,7,0.010945600271224976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,63,0.01098880022764206
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,127,0.011262399703264236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,255,0.015136000514030457
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,511,0.016633599996566772
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,1023,0.020729599893093108
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,2047,0.025591999292373657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,4095,0.03355840146541596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,8191,0.05304480195045471
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,1,0.01127680018544197
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,7,0.01125119999051094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,3,0.011273600161075592
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,15,0.011287999898195266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,31,0.011345600336790084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,63,0.011419200152158738
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,127,0.01162400022149086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,255,0.015438400208950043
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,511,0.021996800601482392
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,63,0.010976000130176545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,1023,0.026612800359725953
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,2047,0.034327998757362366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,4095,0.05448639988899231
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,8191,0.2170880079269409
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,1,0.011591999977827071
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,16383,0.3994256019592285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,3,0.011318399757146835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,7,0.011582399904727935
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,31,0.01143840029835701
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,1,0.011055999994277954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,63,0.011820799857378005
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,127,0.0118367999792099
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,15,0.011404799669981003
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,511,0.026236799359321595
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,1023,0.03366079926490784
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,511,0.015740799903869628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,2047,0.05409600138664246
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,1,0.011606399714946748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,4095,0.21646080017089844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,3,0.011660800129175187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,15,0.011667200177907944
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,8191,0.3945199966430664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,8191,0.03406879901885986
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,31,0.011750400066375732
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,7,0.011844799667596818
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,255,0.016355200111865996
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,63,0.011878400295972823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,511,0.03271839916706085
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,16383,0.7580368041992187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,1023,0.0524944007396698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,2047,0.21932799816131593
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,16383,0.21992480754852295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,1,0.017446400225162507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,4095,0.40474557876586914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,7,0.017507199943065644
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,31,0.017556799948215483
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,8191,0.7736815929412841
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,63,0.017875200510025023
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,255,0.026788800954818726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,511,0.05517759919166565
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,127,0.019099199771881105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,1023,0.21563999652862548
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,16383,1.5317872047424317
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,255,0.016278399527072905
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,2047,0.4149871826171875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,1,0.010931199789047242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,3,0.01085439994931221
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,4095,0.7847280025482177
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,7,0.010883200168609618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,31,0.010940799862146378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,63,0.01103999987244606
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,127,0.011193600296974183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,15,0.010838399827480315
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,127,0.012371200323104858
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,255,0.01510079950094223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,2047,0.020759999752044678
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,1023,0.020075200498104094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,4095,0.02189760059118271
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,8191,1.5341279983520508
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,16383,0.03541440069675446
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,8191,0.026734399795532226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,3,0.010876800119876861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,7,0.011129599809646607
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,3,0.01754560023546219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,1,0.010846400260925293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,63,0.01143840029835701
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,15,0.017611199617385866
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,511,0.015705600380897522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,127,0.011128000169992446
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,255,0.015384000539779664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,1023,0.020336000621318816
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,2047,0.02104160040616989
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,4095,0.02602880001068115
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,1,0.011486399918794632
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,16383,0.0539792001247406
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,3,0.011446399986743927
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,7,0.010883200168609618
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,15,0.011559999734163284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,31,0.010865599662065507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,16383,3.1048479080200195
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,127,0.011196800321340562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,511,0.016590400040149687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,1023,0.020670400559902193
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,2047,0.0253711998462677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,255,0.015108799934387207
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,4095,0.03373439908027649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,8191,0.053686398267745974
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,16383,0.22061920166015625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,3,0.010865599662065507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,511,0.015700800716876982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,1,0.01091040000319481
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,15,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,63,0.011068800091743469
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,31,0.011020799726247787
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,127,0.011224000155925751
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,511,0.020822399854660036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,255,0.01520799994468689
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,1023,0.025684800744056702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,31,0.011244799941778183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,4095,0.053260797262191774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,2047,0.033435198664665225
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,8191,0.22018079757690429
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,8191,0.034160000085830686
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,16383,0.4044464111328125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,7,0.012878400087356568
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,3,0.012895999848842621
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,63,0.013177600502967835
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,15,0.012595200538635254
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,255,0.017377600073814392
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,127,0.013817599415779114
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,63,0.011619199812412263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,511,0.03412159979343414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,1023,0.054769599437713624
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,2047,0.2148672103881836
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,4095,0.3975503921508789
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,1,0.018760000169277192
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,7,0.018806399405002595
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,3,0.018478399515151976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,7,0.010924799740314484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,15,0.01886720061302185
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,31,0.018588800728321076
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,8191,0.7624544143676758
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,63,0.01887039989233017
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,255,0.027819201350212097
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,127,0.02024800032377243
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,15,0.01088000014424324
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,1023,0.2130431890487671
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,511,0.05762079954147339
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,2047,0.40352802276611327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,3,0.03059679865837097
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,1,0.012620800733566284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,7,0.030347201228141784
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,31,0.03052160143852234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,4095,0.7596367835998535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,127,0.0324288010597229
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,31,0.012908799946308136
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,255,0.049239999055862425
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,511,0.2145087957382202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,8191,1.4775407791137696
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,1023,0.4093599796295166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,3,0.054204797744750975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,2047,0.7962031841278077
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,1,0.054497599601745605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,15,0.05444959998130798
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,31,0.05474720001220703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,63,0.05516800284385681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,4095,1.5406895637512208
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,127,0.05716800093650818
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,1,0.030345600843429566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,255,0.21779201030731202
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,511,0.40331039428710935
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,15,0.03070879876613617
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,63,0.03091840147972107
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,1,0.019556799530982973
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,3,0.019555200636386872
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,7,0.019539199769496918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,1023,0.7924191951751709
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,15,0.019569599628448488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,63,0.019993600249290467
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,127,0.02101919949054718
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,31,0.01990399956703186
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,255,0.029020801186561584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,511,0.05979999899864197
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,7,0.05432800054550171
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,1023,0.21615839004516602
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,1,0.03163839876651764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,2047,1.6055408477783204
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,3,0.031431999802589414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,7,0.031646400690078735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,2047,0.40882558822631837
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,31,0.03150720000267029
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,63,0.0322816014289856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,15,0.031409600377082826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,255,0.04963200092315674
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,127,0.03374559879302978
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,511,0.21288480758666992
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,1,0.05517920255661011
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,3,0.05522239804267883
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,15,0.05533599853515625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,7,0.05529919862747192
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,1023,0.4248239994049072
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,31,0.05567200183868408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,127,0.05802400112152099
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,63,0.05610880255699158
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,2047,0.7765039920806884
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,4095,3.113068771362305
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,255,0.19999840259552001
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,511,0.40680480003356934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,3,0.09682559967041016
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,8191,3.0044111251831054
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,7,0.096697598695755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,15,0.09711520075798034
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,1,0.09680320024490356
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,63,0.09837599992752075
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,1023,0.7932784080505371
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,127,0.19624320268630982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,31,0.09682719707489014
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,1,0.033995199203491214
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,255,0.4267119884490967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,3,0.03403840065002441
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,511,0.7733151912689209
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,7,0.03416799902915955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,15,0.034020799398422244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,2047,1.6237695693969727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,31,0.03419840037822723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,63,0.034815999865531924
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,127,0.03615359961986542
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,511,0.22037439346313475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,255,0.05270079970359802
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,1,0.05761280059814453
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,3,0.05764480233192444
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,1023,1.561678409576416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,7,0.05756480097770691
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,15,0.05768960118293762
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,1,128,1,bfloat16,bfloat16,1023,0.4404496192932129
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,127,0.061105602979660036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,63,0.0585536003112793
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,255,0.20160319805145263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,511,0.40358881950378417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,1,0.09898560047149658
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,3,0.09880319833755494
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,7,0.09929280281066895
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,31,0.09947839975357056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,63,0.10119999647140503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,15,0.09928159713745117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,127,0.20194718837738038
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,2047,3.1341888427734377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,1023,0.7769983768463135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,255,0.3929728031158447
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,1,0.1823904037475586
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,3,0.18213920593261718
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,511,0.7818655967712402
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,31,0.18312159776687623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,7,0.18301600217819214
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,15,0.18240799903869628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,63,0.22294559478759765
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,127,0.3914128065109253
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,4,128,1,bfloat16,bfloat16,1023,1.5505632400512694
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,255,0.7639120101928711
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,1,0.010932800173759461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,8191,6.140382385253906
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,3,0.010782399773597717
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,2,128,1,bfloat16,bfloat16,31,0.05774239897727966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,31,0.010886400192975997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,255,0.015063999593257904
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,7,0.010956799983978272
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,127,0.011070399731397628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,15,0.010900799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,2047,0.021374399960041045
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,1023,0.020508800446987153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,63,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,8191,0.034273600578308104
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,4095,0.02608320116996765
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,511,1.519379234313965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,3,0.010840000212192535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,31,0.010894399881362916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,16383,0.05445759892463684
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,63,0.01098880022764206
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,255,0.015115199983119965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,127,0.011307200044393539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,511,0.016763199865818024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,15,0.010921599715948105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,1023,0.02088800072669983
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,2047,0.025464001297950744
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,8191,0.053380799293518064
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,4095,0.03398720026016235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,1,0.011006399989128113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,7,0.010990399867296219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,15,0.010942400246858597
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,31,0.010995200276374817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,3,0.01088000014424324
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,16,8,128,1,bfloat16,bfloat16,1023,3.0533504486083984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,63,0.011097600311040878
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,127,0.011243200302124024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,511,0.020980800688266753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,1023,0.02566399872303009
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,2047,0.03347519934177399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,4095,0.05306400060653686
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,255,0.015164799988269806
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,1,0.01104160025715828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,3,0.011081600189208984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,16383,0.40560641288757326
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,7,0.011032000184059143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,511,0.015881599485874177
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,31,0.011107199639081956
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,15,0.011033599823713302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,63,0.01149279996752739
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,255,0.015432000160217285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,127,0.01157120019197464
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,1,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,7,0.010860799998044967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,2047,0.0533407986164093
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,8191,0.4096047878265381
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,1,0.05792160034179687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,16383,0.214353609085083
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,3,0.05799199938774109
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,15,0.05788959860801697
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,7,0.05782399773597717
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,31,0.058019202947616574
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,63,0.058911997079849246
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,16383,0.778991985321045
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,127,0.06129119992256164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,255,0.21139678955078126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,1,0.09928640127182006
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,7,0.0994912028312683
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,3,0.09905599951744079
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,8191,0.221343994140625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,1,128,1,bfloat16,bfloat16,511,0.4144464015960693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,15,0.09952160120010375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,31,0.0998416006565094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,63,0.10163040161132812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,127,0.20580320358276366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,255,0.3907311916351318
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,3,0.18261439800262452
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,1023,0.033283200860023496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,7,0.18303519487380981
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,2,128,1,bfloat16,bfloat16,511,0.7738239765167236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,15,0.18256800174713134
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,31,0.18332480192184447
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,63,0.22206718921661378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,4095,0.2384943962097168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,127,0.40015358924865724
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,1,0.3536479949951172
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,255,0.7726480007171631
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,3,0.35455200672149656
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,7,0.35484321117401124
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,15,0.35386559963226316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,31,0.35543839931488036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,63,0.4126255989074707
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,511,1.5322688102722168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,3,0.09992640018463135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,127,0.7579967975616455
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,1,0.10005279779434204
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,15,0.09988160133361816
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,7,0.09977120161056519
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,63,0.10262240171432495
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,4,128,1,bfloat16,bfloat16,1,0.18271679878234864
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,31,0.10022560358047486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,127,0.22103679180145264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,255,1.5115376472473145
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,3,0.18344160318374633
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,7,0.18285919427871705
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,1,0.1837488055229187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,15,0.18355679512023926
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,1,128,1,bfloat16,bfloat16,255,0.41830878257751464
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,31,0.18369280099868773
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,1,0.3536607980728149
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,127,0.400270414352417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,255,0.7702271938323975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,7,0.35444159507751466
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,3,0.3534895896911621
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,15,0.3541552066802979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,63,0.4329792022705078
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,31,0.3594304084777832
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,16,8,128,1,bfloat16,bfloat16,511,3.0049264907836912
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,127,0.773360013961792
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,1,0.6943103790283203
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,3,0.6954527854919433
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,7,0.6964655876159668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,4,128,1,bfloat16,bfloat16,255,1.5336848258972169
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,15,0.6956448078155517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,31,0.7253583908081055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,511,0.025756800174713136
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,1,0.01096320003271103
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,63,0.8095904350280761
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,3,0.011004800349473954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,15,0.01096000000834465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,63,0.011155200004577637
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,127,0.011180800199508668
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,255,0.015227200090885162
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,2,128,1,bfloat16,bfloat16,63,0.2334912061691284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,31,0.010924799740314484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,1023,0.021191999316215515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,8191,0.05410720109939575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,2047,0.026281601190567015
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,4095,0.03413119912147522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,3,0.010950399935245514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,1,0.011484800279140473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,7,0.010953599959611893
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,16383,0.21691360473632812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,15,0.011521600186824799
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,127,1.528596782684326
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,63,0.011110399663448334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,511,0.02106720060110092
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,1023,0.0257968008518219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,255,0.015175999701023101
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,4095,0.05376480221748352
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,8191,0.21722240447998048
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,7,0.011660800129175187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,16,8,128,1,bfloat16,bfloat16,255,3.100579261779785
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,1,0.011687999963760376
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,511,0.017129600048065186
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,7,0.0118367999792099
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,15,0.011028800159692764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,16383,0.39398560523986814
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,3,0.011030399799346923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,31,0.011664000153541566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,127,0.01154239997267723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,255,0.015408000349998474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,1023,0.03326080143451691
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,31,0.010979200154542923
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,2047,0.053819197416305545
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,4095,0.221779203414917
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,127,0.011619199812412263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,1,0.011235199868679047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,7,0.011340799927711486
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,8191,0.40528478622436526
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,15,0.011515200138092041
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,2047,0.033795198798179625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,63,0.011563199758529662
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,127,0.01213119998574257
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,31,0.011311999708414077
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,255,0.016076800227165223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,511,0.03189919888973236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,1023,0.051974397897720334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,2047,0.21807680130004883
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,16383,0.8574704170227051
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,4095,0.4009696006774902
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,1,0.011804799735546111
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,15,0.011828800290822982
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,31,0.011846400052309036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,7,0.011814399808645248
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,63,0.011908800154924393
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,127,0.012238399684429168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,255,0.016148799657821657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,511,0.026628801226615907
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,1023,0.03458879888057709
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,2047,0.05475519895553589
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,63,0.011264000087976456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,4095,0.21595680713653564
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,511,0.025740799307823182
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,8191,0.39935998916625975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,16383,1.531099224090576
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,3,0.011999999731779098
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,16383,0.7672416210174561
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,3,0.011695999652147293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,63,0.012627199292182922
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,31,0.012200000137090683
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,127,0.013096000254154205
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,511,0.03358719944953918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,255,0.0169295996427536
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,32767,1.4873071670532227
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,1023,0.053513598442077634
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,2047,0.2134848117828369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,3,0.01173280030488968
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,8191,0.7726016044616699
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,4095,0.42517762184143065
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,1,0.017900800704956053
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,8191,0.7531744003295898
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,3,0.017969599366188048
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,15,0.01801760047674179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,31,0.01805119961500168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,7,0.018113599717617036
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,1,0.012329600006341934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,127,0.019577600061893463
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,16383,1.4790703773498535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,15,0.012099199742078782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,7,0.012025599926710128
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,511,0.05618079900741577
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,2047,0.4148543834686279
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,1023,0.23541760444641113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,4095,0.7894192218780518
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,1,0.01072160005569458
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,3,0.010729599744081497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,32767,2.9089120864868163
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,7,0.010711999982595444
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,8191,1.5251328468322753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,31,0.010737600177526474
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,127,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,255,0.01485760062932968
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,511,0.015201599895954132
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,15,0.010740800201892853
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,63,0.010955200344324113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,1023,0.019099199771881105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,8191,0.02158239930868149
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,2047,0.019487999379634857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,16383,0.02431679964065552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,4095,0.020318399369716644
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,1,0.010739199817180634
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,63,0.018246400356292724
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,7,0.01072480008006096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,15,0.010764800012111664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,3,0.010791999846696853
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,31,0.010777600109577179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,63,0.010828799754381179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,255,0.02775999903678894
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,255,0.014921599626541137
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,511,0.015276800096035003
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,2047,0.019571200013160706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,4095,0.02062239944934845
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,8191,0.02229759991168976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,16383,0.027540799975395203
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,32767,0.03623040020465851
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,1,0.010843200236558914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,3,0.010811199992895126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,15,0.010804799944162368
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,31,0.010828799754381179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,16383,3.0190223693847655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,63,0.010911999642848969
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,255,0.01501920074224472
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,511,0.015196800231933594
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,1023,0.01912800073623657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,2047,0.020099200308322906
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,127,0.011131200194358825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,4095,0.021167999505996703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,16383,0.03452000021934509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,8191,0.026044800877571106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,32767,0.05424320101737976
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,1,0.010740800201892853
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,3,0.010705599933862687
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,15,0.010740800201892853
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,7,0.010788799822330475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,31,0.010729599744081497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,127,0.01101119965314865
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,63,0.010958400368690491
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,255,0.014932799339294433
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,511,0.015329599380493164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,2047,0.01961600035429001
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,32767,0.030220800638198854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,16383,0.027496001124382018
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,4095,0.020750400424003602
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,32767,0.036683198809623715
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,127,0.01117279976606369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,1,0.010814400017261505
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,7,0.010806400328874588
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,1023,0.019699199497699736
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,31,0.010892800241708755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,15,0.010815999656915664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,127,0.011115200072526931
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,255,0.015009599924087524
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,511,0.015479999780654907
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,32767,5.983257675170899
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,1023,0.019406400620937347
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,2047,0.020257599651813507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,8191,0.026144000887870788
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,16383,0.03484320044517517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,1,0.010897599905729295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,32767,0.05444480180740356
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,3,0.010867200046777725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,4095,0.02139039933681488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,15,0.010955200344324113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,7,0.010923200100660325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,63,0.011004800349473954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,127,0.011208000034093857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,31,0.010908800363540649
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,255,0.015068799257278442
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,1023,0.02046400010585785
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,511,0.016121600568294526
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,2047,0.021040000021457672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,8191,0.03404479920864105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,4095,0.02547360062599182
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,16383,0.05395039916038513
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,1,0.011281599849462509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,3,0.011307200044393539
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,32767,0.23733439445495605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,1023,0.01932799965143204
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,31,0.011283200234174728
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,63,0.01141439974308014
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,127,0.011555200070142746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,15,0.011249600350856781
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,255,0.015412800014019012
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,8191,0.02250880002975464
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,1023,0.02624320089817047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,511,0.02152319997549057
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,2047,0.03406560122966766
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,3,0.010761599987745285
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,4095,0.05369600057601929
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,63,0.012639999389648438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,7,0.010804799944162368
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,16383,0.39868319034576416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,8191,0.2333967924118042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,3,0.011713600158691407
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,7,0.014241600036621093
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,63,0.014001600444316864
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,15,0.011392000317573547
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,127,0.013233600556850434
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,255,0.016073599457740784
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,511,0.026020801067352294
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,32767,0.8940303802490235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,1023,0.03437280058860779
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,4095,0.21622560024261475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,2047,0.05419359803199768
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,1,0.011959999799728394
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,8191,0.3942080020904541
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,3,0.012268800288438797
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,15,0.011958400160074234
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,16383,0.7577888011932373
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,7,0.011939200013875962
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,31,0.012001600116491318
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,127,0.012705600261688233
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,255,0.016752000153064727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,63,0.012276799976825714
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,511,0.03270080089569092
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,1023,0.05288000106811523
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,32767,1.474947166442871
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,2047,0.2207263946533203
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,4095,0.40446882247924804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,1,0.011350400000810622
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,3,0.011139199882745743
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,1,0.010952000319957734
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,7,0.01119839996099472
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,15,0.010788799822330475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,8191,0.7710256099700927
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,63,0.010950399935245514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,255,0.015009599924087524
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,511,0.015409600734710694
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,1023,0.019625599682331085
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,127,0.011182399839162827
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,2047,0.02033279985189438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,8191,0.026208001375198364
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,4095,0.02147040069103241
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,16383,1.5315759658813477
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,16383,0.03507519960403442
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,32767,0.054476797580718994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,3,0.010831999778747558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,15,0.010862399637699128
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,7,0.010876800119876861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,1,0.010859200358390808
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,31,0.010920000076293946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,63,0.010985600203275681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,255,0.015039999783039094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,511,0.015659199655056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,127,0.011187200248241425
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,1023,0.02025440037250519
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,2047,0.021065600216388702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,7,0.011281599849462509
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,4095,0.025566399097442627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,16383,0.05355520248413086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,32767,0.21334559917449952
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,3,0.010924799740314484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,31,0.01093600019812584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,7,0.010974399745464325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,63,0.011076799780130386
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,127,0.011188799887895584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,15,0.011147200316190719
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,255,0.015143999457359314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,511,0.016633599996566772
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,2047,0.02556000053882599
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,4095,0.03363519906997681
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,32767,3.0129600524902345
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,8191,0.053308802843093875
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,1,0.01255200058221817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,16383,0.22118721008300782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,3,0.012510399520397186
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,7,0.01255200058221817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,31,0.011448000371456147
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,31,0.011139199882745743
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,15,0.012547199428081513
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,31,0.012848000228404998
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,63,0.013152000308036805
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,255,0.017478400468826295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,511,0.03441280126571655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,1023,0.05420799851417542
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,2047,0.21435999870300293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,4095,0.39607839584350585
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,8191,0.03404960036277771
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,1,0.018462400138378143
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,1,0.010948800295591355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,3,0.018515199422836304
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,7,0.018505600094795228
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,15,0.018532800674438476
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,8191,0.7601408004760742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,31,0.018555200099945067
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,63,0.018937599658966065
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,255,0.027952000498771667
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,1023,0.020937600731849672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,127,0.020047999918460846
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,1023,0.21085920333862304
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,2047,0.4020847797393799
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,16383,1.4981648445129394
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,32767,0.44324479103088377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,127,0.013305599987506866
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,4095,0.75830078125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,7,0.030369600653648375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,3,0.030299198627471925
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,31,0.03049759864807129
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,1,0.03025439977645874
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,127,0.0324319988489151
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,15,0.030351999402046203
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,63,0.031040000915527343
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,255,0.04868800044059753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,511,0.21413760185241698
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,8191,1.475268840789795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,1023,0.4058688163757324
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,2047,0.7925295829772949
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,1,0.019467200338840484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,7,0.01951040029525757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,15,0.019460800290107726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,31,0.01955360025167465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,3,0.01953279972076416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,63,0.0199535995721817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,511,0.056852799654006955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,127,0.020927999913692475
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,255,0.02895520031452179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,511,0.059543997049331665
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,4095,1.531495952606201
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,1023,0.21437439918518067
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,2047,0.4063888072967529
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,16383,2.8984912872314452
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,3,0.0313264012336731
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,1,0.03127999901771546
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,31,0.031569600105285645
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,7,0.03142400085926056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,127,0.03351520001888275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,63,0.031963199377059937
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,15,0.03153600096702576
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,255,0.049644801020622256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,4095,0.7699359893798828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,511,0.2110447883605957
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,3,0.05496159791946411
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,8191,3.0061920166015623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,1,0.0555840015411377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,7,0.05517119765281677
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,1023,0.4044511795043945
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,63,0.05600799918174744
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,31,0.0559391975402832
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,127,0.05799520015716553
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,255,0.1973744034767151
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,511,0.40405120849609377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,4095,1.4914128303527832
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,1023,0.7919439792633056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,3,0.033995199203491214
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,1,0.03391839861869812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,15,0.03402239978313446
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,63,0.03461120128631592
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,7,0.033990401029586795
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,31,0.034088000655174255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,127,0.03617120087146759
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,255,0.05257599949836731
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,2047,1.601161575317383
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,511,0.2283344030380249
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,1023,0.4031775951385498
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,3,0.057443201541900635
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,1,128,1,bfloat16,bfloat16,2047,0.7792704105377197
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,15,0.0574288010597229
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,1,0.0574895977973938
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,7,0.057625597715377806
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,63,0.058455997705459596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,16383,5.980460739135742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,127,0.06079999804496765
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,31,0.05781919956207275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,255,0.19739359617233276
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,15,0.055086398124694826
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,1,0.09905599951744079
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,3,0.09887520074844361
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,7,0.09921280145645142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,31,0.09969279766082764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,63,0.10136959552764893
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,4095,3.1166080474853515
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,127,0.19805920124053955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,1023,0.831270408630371
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,2047,1.5525615692138672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,255,0.42813758850097655
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,1,0.010809600353240967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,511,0.7756239891052246
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,3,0.01080000028014183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,15,0.010785599797964096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,7,0.010831999778747558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,31,0.010875199735164643
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,63,0.010982400178909302
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,127,0.011166399717330933
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,255,0.015081599354743958
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,1023,1.6964767456054688
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,511,0.01568479984998703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,2047,0.8311807632446289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,2,128,1,bfloat16,bfloat16,511,0.3993439912796021
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,4095,0.025672000646591187
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,15,0.09906880259513855
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,16383,0.053655999898910525
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,1,0.01088479980826378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,8191,0.03446080088615418
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,7,0.010900799930095673
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,32767,0.21523680686950683
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,3,0.010831999778747558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,31,0.010948800295591355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,127,0.011188799887895584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,255,0.015119999647140503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,15,0.011137600243091583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,63,0.010995200276374817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,511,0.01669279932975769
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,1023,0.02089280039072037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,2047,0.02582719922065735
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,4095,0.03373759984970093
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,8191,0.05310400128364563
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,1,0.010958400368690491
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,16383,0.21259839534759523
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,3,0.010924799740314484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,7,0.010966400057077408
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,31,0.01096159964799881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,63,0.011151999980211259
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,15,0.010967999696731567
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,127,0.0112015999853611
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,255,0.015185600519180298
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,32767,0.3919856071472168
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,511,0.02096319943666458
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,1023,0.025595200061798096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,2047,0.03349280059337616
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,8,4,128,1,bfloat16,bfloat16,2047,3.222574234008789
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,4095,0.05387679934501648
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,8191,0.2218832015991211
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,16383,0.4070735931396484
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,1,0.057817602157592775
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,7,0.057734400033950806
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,63,0.05872480273246765
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,15,0.0577567994594574
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,127,0.06101599931716919
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,1023,0.02035360038280487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,31,0.05788800120353699
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,32767,0.9097904205322266
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,255,0.20510399341583252
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,2047,0.021147200465202333
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,1,0.09895840287208557
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,3,0.09938079714775086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,15,0.09908000230789185
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,7,0.09925119876861573
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,511,0.40610718727111816
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,63,0.10179519653320312
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,127,0.1994096040725708
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,1023,0.7831376075744629
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,31,0.10011039972305298
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,255,0.3816879987716675
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,1,0.18243999481201173
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,7,0.18291360139846802
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,3,0.18244800567626954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,15,0.18237600326538086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,31,0.18324639797210693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,511,0.7618783950805664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,63,0.216812801361084
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,127,0.3864991903305054
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,2,128,1,bfloat16,bfloat16,1023,1.5078304290771485
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,3,0.0993664026260376
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,1,0.09941920042037963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,255,0.762721586227417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,7,0.09945759773254395
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,31,0.09983680248260499
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,63,0.10126880407333375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,15,0.0998207986354828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,127,0.22629759311676026
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,1,0.18236000537872316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,255,0.39604160785675047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,511,1.5171695709228517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,3,0.1829535961151123
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,15,0.18254879713058472
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,31,0.18335200548171998
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,63,0.23665599822998046
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,127,0.3886624097824097
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,255,0.7505504131317139
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,1,0.3547888040542603
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,4,128,1,bfloat16,bfloat16,1023,3.057547187805176
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,3,0.35355520248413086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,7,0.35547680854797364
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,31,0.3559247970581055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,511,1.4914671897888183
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,63,0.4273519992828369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,8,1,128,1,bfloat16,bfloat16,3,0.05812479853630066
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,15,0.3543551921844482
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,1,0.011168000102043153
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,3,0.010918399691581726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,7,0.01114879995584488
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,31,0.011116799712181092
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,15,0.010918399691581726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,63,0.011243200302124024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,127,0.011217600107192994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,2,128,1,bfloat16,bfloat16,7,0.18258240222930908
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,255,0.015311999619007111
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,1023,0.02091200053691864
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,1,128,1,bfloat16,bfloat16,511,0.778326416015625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,511,0.016897599399089813
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,2047,0.025854399800300597
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,4095,0.03365919888019562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,1,0.010918399691581726
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,3,0.010886400192975997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,255,1.5114591598510743
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,8191,0.05353760123252869
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,7,0.010947199910879135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,16383,0.23334879875183107
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,63,0.011059200018644333
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,31,0.01096320003271103
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,255,0.015131199359893798
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,15,0.010974399745464325
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,511,0.020947200059890748
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,1023,0.025763198733329773
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,127,0.011300799995660782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,4095,0.05327519774436951
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,2047,0.03373599946498871
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,8191,0.21559040546417235
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,16383,0.39058239459991456
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,3,0.010983999818563461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,1,0.01106399968266487
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,15,0.011086399853229522
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,63,0.011191999912261963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,31,0.011108800023794174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,127,0.011473599821329117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,7,0.011049599945545196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,255,0.015440000593662262
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,511,0.025436800718307496
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,1023,0.03327359855175018
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,511,3.0009408950805665
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,32767,0.8864831924438477
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,2047,0.05321919918060303
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,4095,0.2218400001525879
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,8,4,128,1,bfloat16,bfloat16,127,0.7594719886779785
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,8191,0.40579519271850584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,3,0.011673600226640702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,31,0.011721599847078323
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,63,0.011870399862527848
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,7,0.01175519973039627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,15,0.011750400066375732
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,127,0.012164799869060517
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,511,0.027249601483345032
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,255,0.01613599956035614
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,1023,0.03506399989128113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,2047,0.05547040104866028
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,16383,0.8559776306152344
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,4095,0.21776959896087647
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,32767,0.3995136022567749
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,8191,0.3988944053649902
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,32767,1.5465408325195313
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,1,0.01218400001525879
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,3,0.01197120025753975
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,16383,0.7689919948577881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,15,0.012307199835777282
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,31,0.012030400335788727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,255,0.01672479957342148
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,7,0.011998400092124939
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,127,0.01295360028743744
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,511,0.033313599228858945
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,63,0.012252800166606903
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,1023,0.053406399488449094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,2047,0.21360158920288086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,32767,1.4907872200012207
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,4095,0.3935807943344116
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,1,0.011673600226640702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,7,0.010751999914646149
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,8191,0.7514768123626709
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,15,0.010780800133943558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,63,0.010876800119876861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,127,0.011102399975061416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,255,0.014926399290561675
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,31,0.010676799714565277
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,511,0.01600639969110489
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,2047,0.02021919935941696
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,1023,0.01992959976196289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,4095,0.02051520049571991
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,8191,0.02212799936532974
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,16383,1.4789551734924316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,16383,0.025561600923538208
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,65535,0.0423088014125824
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,65535,2.928868865966797
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,1,0.010686399787664414
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,32767,0.03095200061798096
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,7,0.010715200006961823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,31,0.010771200060844421
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,15,0.010830400139093399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,63,0.010828799754381179
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,3,0.010913600027561188
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,127,0.011158400028944016
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,511,0.016212800145149232
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,1023,0.01998399943113327
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,2047,0.02029760032892227
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,4095,0.021111999452114106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,16383,0.02746239900588989
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,8191,0.023054400086402894
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,32767,0.0363103985786438
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,32767,2.903817558288574
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,1,0.010822399705648422
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,65535,0.05565599799156189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,7,0.010862399637699128
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,31,0.010847999900579452
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,63,0.01096000000834465
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,3,0.0108815997838974
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,1,0.010684800148010255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,127,0.011204800009727478
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,3,0.010758399963378906
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,255,0.014948800206184387
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,511,0.01600960046052933
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,1023,0.02003999948501587
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,2047,0.020164799690246583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,8191,0.02289759963750839
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,16383,0.029547199606895447
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,4095,0.021155199408531188
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,32767,0.03824160099029541
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,1,0.01080320030450821
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,65535,0.058019202947616574
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,3,0.01080159991979599
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,7,0.010867200046777725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,127,0.011169599741697312
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,63,0.010897599905729295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,255,0.014983999729156493
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,1023,0.020212799310684204
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,511,0.01613280028104782
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,2047,0.020763200521469117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,8191,0.027243199944496154
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,4095,0.021639999747276307
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,16383,0.03572799861431122
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,32767,0.055297601222991946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,255,0.014886400103569031
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,65535,0.21377279758453369
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,3,0.011214400082826615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,15,0.011246400326490403
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,7,0.011262399703264236
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,31,0.011297599971294403
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,1,0.011255999654531479
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,63,0.011363200098276138
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,127,0.011555200070142746
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,65535,5.750487899780273
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,255,0.015436799824237823
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,511,0.0220880001783371
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,1023,0.026953598856925963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,4095,0.05421599745750427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,8191,0.21605439186096193
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,2047,0.034822401404380796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,16383,0.4015952110290527
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,15,0.010830400139093399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,1,0.011443199962377549
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,7,0.011377599835395814
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,3,0.011355199664831162
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,15,0.011454399675130844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,31,0.01146719977259636
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,63,0.011559999734163284
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,255,0.015795199573040007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,511,0.026796799898147584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,127,0.011900799721479416
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,1023,0.03455840051174164
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,32767,0.8957360267639161
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,15,0.010892800241708755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,2047,0.05491039752960205
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,31,0.010836800187826156
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,1,0.011073599755764007
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,8191,0.3939023971557617
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,4095,0.22810239791870118
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,3,0.011851199716329575
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,31,0.011097600311040878
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,16383,0.7583824157714844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,65535,1.746433639526367
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,63,0.011190400272607804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,15,0.010867200046777725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,511,0.01632480025291443
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,127,0.011131200194358825
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,1023,0.02025440037250519
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,255,0.015024000406265258
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,2047,0.02112800031900406
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,4095,0.02168159931898117
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,8191,0.02731040120124817
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,32767,1.4759712219238281
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,1,0.010940799862146378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,16383,0.03795360028743744
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,32767,0.05772960186004639
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,7,0.010868799686431885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,65535,0.21802239418029784
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,63,0.010999999940395355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,3,0.01088000014424324
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,15,0.01088000014424324
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,31,0.011727999895811081
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,127,0.011137600243091583
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,511,0.016281600296497344
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,1023,0.021220800280570985
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,2047,0.021924799680709837
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,4095,0.02602880001068115
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,16383,0.05470560193061828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,8191,0.035392001271247864
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,1,0.01247519999742508
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,32767,0.21407999992370605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,7,0.012567999958992004
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,3,0.012566399574279786
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,65535,0.39451839923858645
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,63,0.012801599502563477
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,65535,2.908945655822754
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,15,0.012726399302482604
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,255,0.017318400740623473
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,127,0.01329759955406189
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,1023,0.05438719987869263
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,2047,0.21314239501953125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,4095,0.3959536075592041
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,7,0.010860799998044967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,1,0.018638400733470915
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,3,0.01863040030002594
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,7,0.018622399866580965
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,15,0.018671999871730804
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,31,0.01874080002307892
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,63,0.01905599981546402
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,127,0.02003680020570755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,255,0.015105600655078887
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,255,0.02810400128364563
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,16383,1.4972368240356446
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,511,0.05693920254707337
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,1023,0.2104880094528198
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,31,0.012559999525547028
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,2047,0.43445920944213867
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,4095,0.7581183910369873
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,511,0.033932799100875856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,1,0.019515199959278105
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,3,0.019472000002861024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,7,0.019472000002861024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,32767,2.953555107116699
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,15,0.019497600197792054
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,31,0.01961120069026947
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,255,0.0347216010093689
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,8191,1.4828800201416015
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,127,0.020870399475097657
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,8191,0.7621007919311523
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,511,0.058684802055358885
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,2047,0.4050159931182861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,1023,0.23146400451660157
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,1,0.03131679892539978
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,4095,0.768830394744873
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,3,0.03139199912548065
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,15,0.031376001238822934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,63,0.03197920024394989
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,16383,2.9008655548095703
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,127,0.03343839943408966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,8191,1.4955327987670899
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,255,0.049737599492073056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,511,0.2097951889038086
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,1023,0.42445597648620603
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,2047,0.8324560165405274
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,63,0.019991999864578246
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,3,0.0338703989982605
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,1,0.03394399881362915
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,7,0.033904001116752625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,15,0.033924800157546994
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,31,0.034108799695968625
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,4095,1.4921888351440429
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,127,0.03603039979934693
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,63,0.034646400809288026
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,7,0.0313264012336731
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,255,0.05257279872894287
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,511,0.21535201072692872
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,31,0.031585600972175595
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,32767,5.750446319580078
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,3,0.05743839740753174
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,1,0.05742719769477844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,1023,0.42991042137145996
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,15,0.05755839943885803
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,31,0.05768960118293762
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,2047,0.7781760215759277
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,7,0.05747519731521607
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,63,0.058433598279953
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,127,0.060659199953079224
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,255,0.19537919759750366
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,3,0.010831999778747558
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,8191,2.9147167205810547
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,7,0.010886400192975997
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,511,0.39979679584503175
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,31,0.010902400314807891
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,1023,0.7701312065124511
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,63,0.010947199910879135
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,1,128,1,bfloat16,bfloat16,4095,1.5059120178222656
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,255,0.01499519944190979
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,15,0.01090560033917427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,1023,0.021076799929142
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,127,0.011191999912261963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,4095,0.026043200492858888
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,2047,0.021686400473117828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,511,0.016312000155448914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,16383,0.05704479813575745
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,8191,0.03567039966583252
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,3,0.010937599837779999
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,1,0.010857599973678588
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,7,0.010931199789047242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,2047,1.6513711929321289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,15,0.010871999710798264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,31,0.010977599769830704
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,127,0.011214400082826615
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,255,0.015147200226783753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,511,0.017428800463676453
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,1023,0.021695999801158904
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,2047,0.026092800498008727
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,63,0.011076799780130386
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,8191,0.05472000241279602
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,16383,0.22995519638061523
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,4,2,128,1,bfloat16,bfloat16,4095,3.0163600921630858
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,32767,0.39255359172821047
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,1,0.010849600285291671
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,3,0.05768479704856873
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,7,0.05820000171661377
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,1,0.05782719850540161
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,31,0.0583840012550354
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,63,0.058723199367523196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,127,0.06108800172805786
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,65535,0.9986160278320313
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,255,0.21986238956451415
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,511,0.4018671989440918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,32767,0.235481595993042
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,65535,0.44295358657836914
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,1023,0.8515551567077637
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,3,0.09884799718856811
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,1,0.0994592010974884
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,4095,0.0344895988702774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,31,0.09982240200042725
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,7,0.09926239848136902
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,15,0.09941759705543518
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,127,0.19577280282974244
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,255,0.3775376081466675
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,1,0.0992464005947113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,511,0.7546031951904297
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,15,0.05764319896697998
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,3,0.09922879934310913
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,15,0.09927679896354676
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,31,0.09999359846115112
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,63,0.10174239873886108
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,7,0.09948639869689942
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,127,0.20108959674835206
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,255,0.3887120008468628
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,1023,1.6145408630371094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,511,0.7667119979858399
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,1,0.18253599405288695
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,3,0.18257440328598024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,15,0.18262399435043336
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,7,0.1829967975616455
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,31,0.18343039751052856
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,63,0.10159679651260375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,1,128,1,bfloat16,bfloat16,1023,1.5171456336975098
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,2,128,1,bfloat16,bfloat16,2047,3.040768051147461
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,255,0.7423984050750733
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,1,0.010931199789047242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,3,0.010950399935245514
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,15,0.01093600019812584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,4,1,128,1,bfloat16,bfloat16,2047,1.6001663208007812
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,31,0.010972800105810166
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,63,0.011049599945545196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,7,0.010846400260925293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,255,0.015119999647140503
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,511,0.01751679927110672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,1023,0.021611200273036958
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,127,0.01111999973654747
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,2047,0.026305601000785828
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,4095,0.0342848002910614
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,8191,0.05488160252571106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,1,0.010929600149393082
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,3,0.010920000076293946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,7,0.010892800241708755
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,16383,0.23539199829101562
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,15,0.010920000076293946
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,32767,0.44036478996276857
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,63,0.011084800213575363
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,255,0.015113599598407745
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,511,0.02171040028333664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,31,0.01096159964799881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,127,0.011294399946928024
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,65535,0.9365839958190918
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,2047,0.03416799902915955
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,4095,0.05421760082244873
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,1023,2.9719024658203126
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,8191,0.22271039485931396
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,63,0.232806396484375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,16383,0.3953536033630371
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,1,0.011723200231790543
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,3,0.011697600036859513
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,15,0.011751999706029892
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,31,0.011812800168991089
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,63,0.011908800154924393
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,127,0.012216000258922577
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,32767,0.887600040435791
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,255,0.01608159989118576
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,1023,0.03560000061988831
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,511,1.4789551734924316
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,511,0.02773439884185791
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,2047,0.05532640218734741
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,65535,1.7338159561157227
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,8191,0.4001152038574219
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,16383,0.7692527770996094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,1023,0.026390400528907777
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,1,0.010667199641466141
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,3,0.010694400221109391
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,7,0.01067039966583252
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,15,0.010664000362157821
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,31,0.010700800269842149
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,4,2,128,1,bfloat16,bfloat16,127,0.3834127902984619
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,32767,1.488868808746338
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,7,0.011740799993276596
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,127,0.011206399649381638
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,511,0.0164560005068779
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,1023,0.020483200252056123
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,255,0.014959999918937683
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,2047,0.02048480063676834
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,4095,0.020716799795627593
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,8191,0.02263679951429367
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,16383,0.02645600140094757
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,32767,0.031142398715019226
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,65535,0.042587199807167055
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,1,0.01093600019812584
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,131071,0.06253439784049988
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,7,0.010999999940395355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,15,0.011057599633932113
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,31,0.010998400300741196
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,63,0.011095999926328658
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,127,0.011255999654531479
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,255,0.015177600085735321
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,511,0.016540800034999848
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,4095,0.23575520515441895
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,65535,2.9327791213989256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,1023,0.020627200603485107
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,2047,0.020638400316238405
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,4095,0.021111999452114106
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,16383,0.030694401264190672
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,32767,0.03930880129337311
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,65535,0.05898399949073792
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,1,0.011454399675130844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,3,0.011377599835395814
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,131071,0.21939361095428467
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,15,0.01143840029835701
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,63,0.01154239997267723
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,31,0.011481600254774094
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,63,0.010784000158309937
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,255,0.015483200550079346
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,127,0.011739200353622437
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,511,0.022745600342750548
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,1023,0.0271807998418808
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,2047,0.03523359894752502
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,4095,0.055579197406768796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,8191,0.21759359836578368
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,16383,0.4017439842224121
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,3,0.010840000212192535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,32767,0.895695972442627
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,1,0.010809600353240967
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,15,0.010835199803113937
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,3,0.010876800119876861
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,31,0.010824000090360641
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,131071,5.812681579589844
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,127,0.011110399663448334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,7,0.011177600175142289
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,511,0.016439999639987945
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,63,0.010931199789047242
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,2047,0.02104160040616989
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,4095,0.021879999339580535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,65535,1.745467185974121
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,8191,0.027625599503517152
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,8191,0.02335679978132248
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,7,0.011579199880361556
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,16383,0.03880159854888916
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,1,0.012801599502563477
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,65535,0.21813600063323973
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,32767,0.058513599634170535
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,7,0.0127920001745224
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,131071,0.40226240158081056
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,3,0.012569600343704223
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,63,0.013092799484729767
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,31,0.012559999525547028
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,255,0.017423999309539796
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,511,0.03426879942417145
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,1023,0.053904002904891966
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,2047,0.21337440013885497
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,4095,0.39794719219207764
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,131071,3.454598236083984
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,8191,0.7601744174957276
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,1023,0.020561599731445314
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,16383,1.4999600410461427
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,255,0.015044799447059632
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,1,0.019734400510787963
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,7,0.01945279985666275
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,15,0.019791999459266664
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,3,0.02791680097579956
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,63,0.02016319930553436
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,15,0.012564800679683685
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,31,0.01958400011062622
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,127,0.020904000103473663
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,127,0.013391999900341034
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,255,0.029300799965858458
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,511,0.05892159938812256
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,1023,0.23103199005126954
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,2047,0.42607841491699217
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,32767,2.937455940246582
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,4095,0.769646406173706
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,1,0.03388319909572601
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,31,0.03408159911632538
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,3,0.034004798531532286
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,63,0.03459360003471375
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,15,0.03423359990119934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,127,0.03598400056362152
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,7,0.03409439921379089
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,255,0.05230240225791931
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,8191,1.4966336250305177
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,511,0.21496639251708985
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,1023,0.4004240036010742
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,1,0.011099199950695037
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,3,0.010875199735164643
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,7,0.011112000048160552
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,15,0.01085439994931221
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,31,0.01106719970703125
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,2047,0.8512384414672851
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,127,0.011366400122642516
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,511,0.01672320067882538
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,1023,0.021305599808692934
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,2047,0.021883200109004974
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,16383,2.9396783828735353
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,4095,1.5070528030395507
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,4095,0.026254400610923767
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,32767,0.21914560794830323
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,8191,0.03605439960956573
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,16383,0.05880159735679626
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,65535,5.824478530883789
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,1,0.05772160291671753
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,65535,0.4013679981231689
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,7,0.057766401767730714
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,3,0.05770559906959534
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,31,0.05803520083427429
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,131071,0.8487039566040039
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,127,0.060952001810073854
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,63,0.05887519717216492
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,512,1,2,1,128,1,bfloat16,bfloat16,8191,2.9854143142700194
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,255,0.2188271999359131
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,511,0.3985663890838623
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,63,0.010958400368690491
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,3,0.09914720058441162
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,1,0.09927999973297119
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,7,0.09939039945602417
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,15,0.09940479993820191
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,1023,0.851534366607666
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,255,0.015036800503730774
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,31,0.09974240064620972
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,63,0.10163520574569702
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,127,0.21652638912200928
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,255,0.3843168020248413
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,2047,1.7032415390014648
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,1,0.010953599959611893
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,511,0.760916805267334
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,3,0.01088479980826378
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,15,0.010897599905729295
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,63,0.01096159964799881
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,7,0.010871999710798264
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,255,0.015035200119018554
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,127,0.011222399771213531
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,31,0.010948800295591355
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,1023,0.0219200000166893
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,511,0.017710399627685548
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,15,0.05788480043411255
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,2047,0.026374399662017822
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,8191,0.05599200129508972
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,4095,0.034753599762916566
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,32767,0.40180001258850095
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,16383,0.23761439323425293
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,1024,1,2,1,128,1,bfloat16,bfloat16,4095,3.039625549316406
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,65535,1.0114784240722656
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,131071,1.842870330810547
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,2047,3.050611114501953
SGLang,0.5.9,NVIDIA L40S,generation_attention,flash_attention,2048,1,2,1,128,1,bfloat16,bfloat16,1023,1.5109647750854491
