Skip to content

Commit 003f4fb

Browse files
nvzhihanjnvzhihanj
authored
Update loadgen to 6.0; Bulk update 6.0 checker bits (#2415)
* Update loadgen to 6.0; Bulk update 6.0 checker bits * Fix qwen3vl
1 parent 8999c4d commit 003f4fb

File tree

3 files changed

+127
-85
lines changed

3 files changed

+127
-85
lines changed

‎.github/workflows/build_wheels.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
name: Build loadgen wheels and release them into PYPI
2+
23
on:
34
release:
45
types: [published]

‎loadgen/VERSION.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
5.1.2
1+
6.0.0

‎tools/submission/submission_checker.py‎

Lines changed: 125 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -25,82 +25,100 @@
2525
submission_checker_dir = os.path.dirname(os.path.realpath(__file__))
2626

2727
MODEL_CONFIG = {
28-
"v5.0": {
28+
"v6.0": {
2929
"models": [
3030
"resnet",
3131
"retinanet",
3232
"bert-99",
3333
"bert-99.9",
34-
"dlrm-v2-99",
35-
"dlrm-v2-99.9",
3634
"3d-unet-99",
3735
"3d-unet-99.9",
38-
"gptj-99",
39-
"gptj-99.9",
36+
"llama3.1-8b",
37+
"llama3.1-8b-edge",
4038
"llama2-70b-99",
4139
"llama2-70b-99.9",
42-
"llama2-70b-interactive-99",
43-
"llama2-70b-interactive-99.9",
4440
"stable-diffusion-xl",
4541
"mixtral-8x7b",
4642
"llama3.1-405b",
4743
"rgat",
4844
"pointpainting",
45+
"deepseek-r1",
46+
"whisper",
47+
"gpt-oss-120b",
48+
"wan-2.2-t2v-a14b",
49+
"qwen3-vl-235b-a22b",
50+
"dlrm-v3",
4951
],
5052
"required-scenarios-datacenter": {
51-
"resnet": ["Server", "Offline"],
5253
"retinanet": ["Server", "Offline"],
53-
"dlrm-v2-99": ["Server", "Offline"],
54-
"dlrm-v2-99.9": ["Server", "Offline"],
54+
"dlrm-v3": ["Server", "Offline"],
5555
"3d-unet-99": ["Offline"],
5656
"3d-unet-99.9": ["Offline"],
57-
"gptj-99": ["Server", "Offline"],
58-
"gptj-99.9": ["Server", "Offline"],
59-
"llama2-70b-99": ["Server", "Offline"],
60-
"llama2-70b-99.9": ["Server", "Offline"],
61-
"llama2-70b-interactive-99": ["Server", "Offline"],
62-
"llama2-70b-interactive-99.9": ["Server", "Offline"],
57+
"llama3.1-8b": ["Offline"],
58+
"llama2-70b-99": ["Offline"],
59+
"llama2-70b-99.9": ["Offline"],
6360
"stable-diffusion-xl": ["Server", "Offline"],
6461
"mixtral-8x7b": ["Server", "Offline"],
65-
"llama3.1-405b": ["Server", "Offline"],
62+
"llama3.1-405b": ["Offline"],
6663
"rgat": ["Offline"],
64+
"whisper": ["Offline"],
65+
"deepseek-r1": ["Offline"],
66+
"gpt-oss-120b": ["Offline"],
67+
"qwen3-vl-235b-a22b": ["Server", "Offline"],
68+
"dlrm-v3": ["Server", "Offline"],
69+
},
70+
"optional-scenarios-datacenter": {
71+
"llama2-70b-99": ["Interactive", "Server"],
72+
"llama2-70b-99.9": ["Interactive", "Server"],
73+
"llama3.1-405b": ["Interactive", "Server"],
74+
"llama3.1-8b": ["Interactive", "Server"],
75+
"deepseek-r1": ["Interactive", "Server"],
76+
"gpt-oss-120b": ["Interactive", "Server"],
6777
},
68-
"optional-scenarios-datacenter": {},
6978
"required-scenarios-edge": {
7079
"resnet": ["SingleStream", "MultiStream", "Offline"],
7180
"retinanet": ["SingleStream", "MultiStream", "Offline"],
7281
"bert-99": ["SingleStream", "Offline"],
7382
"bert-99.9": ["SingleStream", "Offline"],
7483
"3d-unet-99": ["SingleStream", "Offline"],
7584
"3d-unet-99.9": ["SingleStream", "Offline"],
76-
"gptj-99": ["SingleStream", "Offline"],
77-
"gptj-99.9": ["SingleStream", "Offline"],
85+
"llama3.1-8b-edge": ["SingleStream", "Offline"],
7886
"stable-diffusion-xl": ["SingleStream", "Offline"],
7987
"pointpainting": ["SingleStream"],
88+
"whisper": ["Offline"],
8089
},
8190
"optional-scenarios-edge": {},
8291
"required-scenarios-datacenter-edge": {
83-
"resnet": ["SingleStream", "Offline", "MultiStream", "Server"],
92+
"resnet": ["SingleStream", "MultiStream", "Offline", "Server"],
8493
"retinanet": ["SingleStream", "Offline", "MultiStream", "Server"],
8594
"bert-99": ["SingleStream", "Offline"],
8695
"bert-99.9": ["SingleStream", "Offline"],
87-
"dlrm-v2-99": ["Offline", "Server"],
88-
"dlrm-v2-99.9": ["Offline", "Server"],
8996
"3d-unet-99": ["SingleStream", "Offline"],
9097
"3d-unet-99.9": ["SingleStream", "Offline"],
91-
"gptj-99": ["SingleStream", "Offline", "Server"],
92-
"gptj-99.9": ["SingleStream", "Offline", "Server"],
93-
"llama2-70b-99": ["Server", "Offline"],
94-
"llama2-70b-99.9": ["Server", "Offline"],
95-
"llama2-70b-interactive-99": ["Server", "Offline"],
96-
"llama2-70b-interactive-99.9": ["Server", "Offline"],
98+
"llama3.1-8b": ["Offline"],
99+
"llama3.1-8b-edge": ["SingleStream", "Offline"],
100+
"llama2-70b-99": ["Offline"],
101+
"llama2-70b-99.9": ["Offline"],
97102
"stable-diffusion-xl": ["SingleStream", "Offline", "Server"],
98103
"mixtral-8x7b": ["Server", "Offline"],
99-
"llama3.1-405b": ["Server", "Offline"],
104+
"llama3.1-405b": ["Offline"],
100105
"rgat": ["Offline"],
101106
"pointpainting": ["SingleStream"],
107+
"deepseek-r1": ["Offline"],
108+
"whisper": ["Offline"],
109+
"gpt-oss-120b": ["Offline"],
110+
"qwen3-vl-235b-a22b": ["Offline"],
111+
"dlrm-v3": ["Offline", "Server"],
112+
},
113+
"optional-scenarios-datacenter-edge": {
114+
"llama2-70b-99": ["Interactive", "Server"],
115+
"llama2-70b-99.9": ["Interactive", "Server"],
116+
"llama3.1-405b": ["Interactive", "Server"],
117+
"llama3.1-8b": ["Interactive", "Server"],
118+
"deepseek-r1": ["Interactive", "Server"],
119+
"gpt-oss-120b": ["Interactive", "Server"],
120+
"qwen3-vl-235b-a22b": ["Interactive", "Server"],
102121
},
103-
"optional-scenarios-datacenter-edge": {},
104122
"accuracy-target": {
105123
"resnet": ("acc", 76.46 * 0.99),
106124
"retinanet": ("mAP", 37.55 * 0.99),
@@ -111,25 +129,29 @@
111129
"3d-unet-99": ("DICE", 0.86170 * 0.99),
112130
"3d-unet-99.9": ("DICE", 0.86170 * 0.999),
113131

114-
"gptj-99": (
132+
"llama3.1-8b": (
115133
"ROUGE1",
116-
42.9865 * 0.99,
134+
38.7792 * 0.99,
117135
"ROUGE2",
118-
20.1235 * 0.99,
136+
15.9075 * 0.99,
119137
"ROUGEL",
120-
29.9881 * 0.99,
138+
24.4957 * 0.99,
139+
"ROUGELSUM",
140+
35.793 * 0.99,
121141
"GEN_LEN",
122-
4016878 * 0.9,
142+
8167644 * 0.9,
123143
),
124-
"gptj-99.9": (
144+
"llama3.1-8b-edge": (
125145
"ROUGE1",
126-
42.9865 * 0.999,
146+
39.06 * 0.99,
127147
"ROUGE2",
128-
20.1235 * 0.999,
148+
16.1147 * 0.99,
129149
"ROUGEL",
130-
29.9881 * 0.999,
150+
24.6375 * 0.99,
151+
"ROUGELSUM",
152+
36.124 * 0.99,
131153
"GEN_LEN",
132-
4016878 * 0.9,
154+
3051113 * 0.9,
133155
),
134156
"llama2-70b-99": (
135157
"ROUGE1",
@@ -151,26 +173,6 @@
151173
"TOKENS_PER_SAMPLE",
152174
294.45 * 0.9,
153175
),
154-
"llama2-70b-interactive-99": (
155-
"ROUGE1",
156-
44.4312 * 0.99,
157-
"ROUGE2",
158-
22.0352 * 0.99,
159-
"ROUGEL",
160-
28.6162 * 0.99,
161-
"TOKENS_PER_SAMPLE",
162-
294.45 * 0.9,
163-
),
164-
"llama2-70b-interactive-99.9": (
165-
"ROUGE1",
166-
44.4312 * 0.999,
167-
"ROUGE2",
168-
22.0352 * 0.999,
169-
"ROUGEL",
170-
28.6162 * 0.999,
171-
"TOKENS_PER_SAMPLE",
172-
294.45 * 0.9,
173-
),
174176
"stable-diffusion-xl": (
175177
"CLIP_SCORE",
176178
31.68631873,
@@ -201,6 +203,13 @@
201203
),
202204
"rgat": ("acc", 0.7286 * 0.99),
203205
"pointpainting": ("mAP", 0.5425 * 0.999),
206+
"deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274),
207+
"whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99),
208+
# TODO: Placeholder for now
209+
"gpt-oss-120b": ("exact_match", 83.13 * 0.99),
210+
# TODO: Placeholder for now
211+
"qwen3-vl-235b-a22b": ("F1", 0.7903 * 0.99),
212+
"dlrm-v3": ("AUC", 78.663 * 0.99), # TODO: Placeholder for now
204213
},
205214
"accuracy-upper-limit": {
206215
"stable-diffusion-xl": (
@@ -211,10 +220,13 @@
211220
),
212221
"llama2-70b-99": ("TOKENS_PER_SAMPLE", 294.45 * 1.1),
213222
"llama2-70b-99.9": ("TOKENS_PER_SAMPLE", 294.45 * 1.1),
214-
"llama2-70b-interactive-99": ("TOKENS_PER_SAMPLE", 294.45 * 1.1),
215-
"llama2-70b-interactive-99.9": ("TOKENS_PER_SAMPLE", 294.45 * 1.1),
216223
"mixtral-8x7b": ("TOKENS_PER_SAMPLE", 145.9 * 1.1),
217224
"llama3.1-405b": ("TOKENS_PER_SAMPLE", 684.68 * 1.1),
225+
"llama3.1-8b": ("GEN_LEN", 8167644 * 1.1),
226+
"llama3.1-8b-edge": ("GEN_LEN", 3051113 * 1.1),
227+
"deepseek-r1": ("TOKENS_PER_SAMPLE", 1.1 * 3886.2274),
228+
# TODO: Placeholder for now
229+
"gpt-oss-120b": ("TOKENS_PER_SAMPLE", 1.1 * 9999),
218230
},
219231
"accuracy-delta-perc": {
220232
"stable-diffusion-xl": {"CLIP_SCORE": 1, "FID_SCORE": 2}
@@ -228,17 +240,21 @@
228240
"dlrm-v2-99.9": 204800,
229241
"3d-unet-99": 43,
230242
"3d-unet-99.9": 43,
231-
"gptj-99": 13368,
232-
"gptj-99.9": 13368,
243+
"llama3.1-8b": 13368,
244+
"llama3.1-8b-edge": 5000,
233245
"llama2-70b-99": 24576,
234246
"llama2-70b-99.9": 24576,
235-
"llama2-70b-interactive-99": 24576,
236-
"llama2-70b-interactive-99.9": 24576,
237247
"stable-diffusion-xl": 5000,
238248
"mixtral-8x7b": 15000,
239249
"llama3.1-405b": 8313,
240250
"rgat": 788379,
241251
"pointpainting": 1024,
252+
"deepseek-r1": 4388,
253+
"whisper": 1633,
254+
# TODO: Need to add accuracy sample count checkers as well (4395)
255+
"gpt-oss-120b": 6396,
256+
"qwen3-vl-235b-a22b": 48289,
257+
"dlrm-v3": 34996,
242258
},
243259
"dataset-size": {
244260
"resnet": 50000,
@@ -249,17 +265,21 @@
249265
"dlrm-v2-99.9": 330067,
250266
"3d-unet-99": 43,
251267
"3d-unet-99.9": 43,
252-
"gptj-99": 13368,
253-
"gptj-99.9": 13368,
268+
"llama3.1-8b": 13368,
269+
"llama3.1-8b-edge": 5000,
254270
"llama2-70b-99": 24576,
255271
"llama2-70b-99.9": 24576,
256-
"llama2-70b-interactive-99": 24576,
257-
"llama2-70b-interactive-99.9": 24576,
258272
"stable-diffusion-xl": 5000,
259273
"mixtral-8x7b": 15000,
260274
"llama3.1-405b": 8313,
261275
"rgat": 788379,
262276
"pointpainting": 39987,
277+
"deepseek-r1": 4388,
278+
"whisper": 1633,
279+
# TODO: Need to add accuracy sample count checkers as well (4395)
280+
"gpt-oss-120b": 6396,
281+
"qwen3-vl-235b-a22b": 48289,
282+
"dlrm-v3": 34996,
263283
},
264284
# model_mapping.json is expected in the root directory of the
265285
# submission folder for open submissions and so the below dictionary is
@@ -270,28 +290,31 @@
270290
"mobilenet": "resnet",
271291
"resnet50": "resnet",
272292
"llama3_1-405b": "llama3.1-405b",
293+
"llama3_1-8b": "llama3.1-8b",
294+
"llama3_1-8b-edge": "llama3.1-8b-edge",
273295
},
274296
"seeds": {
275297
# TODO: Update random seeds
276-
"qsl_rng_seed": 6023615788873153749,
277-
"sample_index_rng_seed": 15036839855038426416,
278-
"schedule_rng_seed": 9933818062894767841,
298+
"qsl_rng_seed": 1780908523862526354,
299+
"sample_index_rng_seed": 14771362308971278857,
300+
"schedule_rng_seed": 18209322760996052031,
279301
},
280302
"ignore_errors": [],
281303
"latency-constraint": {
282304
"resnet": {"Server": 15000000},
283305
"retinanet": {"Server": 100000000},
284306
"dlrm-v2-99": {"Server": 60000000},
285307
"dlrm-v2-99.9": {"Server": 60000000},
286-
"gptj-99": {"Server": 20000000000},
287-
"gptj-99.9": {"Server": 20000000000},
308+
"llama3.1-8b": {"Server": 20000000000},
288309
"stable-diffusion-xl": {"Server": 20000000000},
289310
"llama2-70b-99": {"Server": 20000000000},
290311
"llama2-70b-99.9": {"Server": 20000000000},
291-
"llama2-70b-interactive-99": {"Server": 20000000000},
292-
"llama2-70b-interactive-99.9": {"Server": 20000000000},
293312
"mixtral-8x7b": {"Server": 20000000000},
294-
"llama3.1-405b": {"Server": 60000000000}
313+
"llama3.1-405b": {"Server": 60000000000},
314+
"deepseek-r1": {"Server": 60000000000},
315+
"gpt-oss-120b": {"Server": 60000000000},
316+
"qwen3-vl-235b-a22b": {"Server": 60000000000},
317+
"dlrm-v3": {"Server": 60000000000},
295318
},
296319
"min-queries": {
297320
"resnet": {
@@ -312,12 +335,10 @@
312335
"dlrm-v2-99.9": {"Server": 270336, "Offline": 1},
313336
"3d-unet-99": {"SingleStream": 1024, "Offline": 1},
314337
"3d-unet-99.9": {"SingleStream": 1024, "Offline": 1},
315-
"gptj-99": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
316-
"gptj-99.9": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
338+
"llama3.1-8b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
339+
"llama3.1-8b-edge": {"SingleStream": 1024, "Offline": 1},
317340
"llama2-70b-99": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
318341
"llama2-70b-99.9": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
319-
"llama2-70b-interactive-99": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
320-
"llama2-70b-interactive-99.9": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
321342
"stable-diffusion-xl": {
322343
"SingleStream": 1024,
323344
"Server": 270336,
@@ -327,6 +348,11 @@
327348
"llama3.1-405b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
328349
"rgat": {"SingleStream": 1024, "Offline": 1},
329350
"pointpainting": {"SingleStream": 1024},
351+
"deepseek-r1": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
352+
"whisper": {"SingleStream": 1024, "Offline": 1},
353+
"gpt-oss-120b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
354+
"qwen3-vl-235b-a22b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
355+
"dlrm-v3": {"Server": 270336, "Offline": 1},
330356
},
331357
},
332358
"v5.1": {
@@ -719,6 +745,10 @@
719745
"rgat": 788379,
720746
"deepseek-r1": 4388,
721747
"whisper": 1633,
748+
# TODO: Need to add accuracy sample count checkers as well (4395)
749+
"gpt-oss-120b": 6396,
750+
"qwen3-vl-235b-a22b": 9660,
751+
"dlrm-v3": 34996,
722752
}
723753

724754
SCENARIO_MAPPING = {
@@ -873,8 +903,19 @@
873903
"deepseek-r1": {
874904
"Server": {
875905
"ttft": 2000 * 1000000, "tpot": 80 * 1000000
906+
},
907+
"Interactive": {
908+
"ttft": 1500 * 1000000, "tpot": 15 * 1000000
876909
}
877-
}
910+
},
911+
"gpt-oss-120b": {
912+
"Server": {
913+
"ttft": 3000 * 1000000, "tpot": 75 * 1000000
914+
},
915+
"Interactive": {
916+
"ttft": 1500 * 1000000, "tpot": 15 * 1000000
917+
}
918+
},
878919

879920
}
880921

0 commit comments

Comments
 (0)