|
205 | 205 | "pointpainting": ("mAP", 0.5425 * 0.999), |
206 | 206 | "deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274), |
207 | 207 | "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99), |
208 | | - "gpt-oss-120b": ("exact_match", 83.13 * 0.99), # TODO: Placeholder for now |
209 | | - "qwen3-vl-235b-a22b": ("F1", 0.7903 * 0.99), # TODO: Placeholder for now |
210 | | - "dlrm-v3": ("AUC", 78.663 * 0.99), # TODO: Placeholder for now |
| 208 | + # TODO: Placeholder for now |
| 209 | + "gpt-oss-120b": ("exact_match", 83.13 * 0.99), |
| 210 | + # TODO: Placeholder for now |
| 211 | + "qwen3-vl-235b-a22b": ("F1", 0.7903 * 0.99), |
| 212 | + "dlrm-v3": ("AUC", 78.663 * 0.99), # TODO: Placeholder for now |
211 | 213 | }, |
212 | 214 | "accuracy-upper-limit": { |
213 | 215 | "stable-diffusion-xl": ( |
|
223 | 225 | "llama3.1-8b": ("GEN_LEN", 8167644 * 1.1), |
224 | 226 | "llama3.1-8b-edge": ("GEN_LEN", 3051113 * 1.1), |
225 | 227 | "deepseek-r1": ("TOKENS_PER_SAMPLE", 1.1 * 3886.2274), |
226 | | - "gpt-oss-120b": ("TOKENS_PER_SAMPLE", 1.1 * 9999), # TODO: Placeholder for now |
| 228 | + # TODO: Placeholder for now |
| 229 | + "gpt-oss-120b": ("TOKENS_PER_SAMPLE", 1.1 * 9999), |
227 | 230 | }, |
228 | 231 | "accuracy-delta-perc": { |
229 | 232 | "stable-diffusion-xl": {"CLIP_SCORE": 1, "FID_SCORE": 2} |
|
248 | 251 | "pointpainting": 1024, |
249 | 252 | "deepseek-r1": 4388, |
250 | 253 | "whisper": 1633, |
251 | | - "gpt-oss-120b": 6396, # TODO: Need to add accuracy sample count checkers as well (4395) |
| 254 | + # TODO: Need to add accuracy sample count checkers as well (4395) |
| 255 | + "gpt-oss-120b": 6396, |
252 | 256 | "qwen3-vl-235b-a22b": 48289, |
253 | 257 | "dlrm-v3": 34996, |
254 | 258 | }, |
|
272 | 276 | "pointpainting": 39987, |
273 | 277 | "deepseek-r1": 4388, |
274 | 278 | "whisper": 1633, |
275 | | - "gpt-oss-120b": 6396, # TODO: Need to add accuracy sample count checkers as well (4395) |
| 279 | + # TODO: Need to add accuracy sample count checkers as well (4395) |
| 280 | + "gpt-oss-120b": 6396, |
276 | 281 | "qwen3-vl-235b-a22b": 48289, |
277 | 282 | "dlrm-v3": 34996, |
278 | 283 | }, |
|
740 | 745 | "rgat": 788379, |
741 | 746 | "deepseek-r1": 4388, |
742 | 747 | "whisper": 1633, |
743 | | - "gpt-oss-120b": 6396, # TODO: Need to add accuracy sample count checkers as well (4395) |
| 748 | + # TODO: Need to add accuracy sample count checkers as well (4395) |
| 749 | + "gpt-oss-120b": 6396, |
744 | 750 | "qwen3-vl-235b-a22b": 9660, |
745 | 751 | "dlrm-v3": 34996, |
746 | 752 | } |
|
0 commit comments