Diplomovka/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-base.json
2026-11-12 16:51:42 +01:00

3250 lines
105 KiB
JSON

{
"model_name": "models/ministral-base",
"config_general": {
"lighteval_sha": "?",
"num_fewshot_seeds": 1,
"max_samples": null,
"job_id": "0",
"start_time": 883355.287443326,
"end_time": 883520.97395618,
"total_evaluation_time_secondes": "165.68651285406668",
"model_config": {
"model_name": "models/ministral-base",
"generation_parameters": {
"num_blocks": null,
"block_size": null,
"early_stopping": null,
"repetition_penalty": null,
"frequency_penalty": null,
"length_penalty": null,
"presence_penalty": null,
"max_new_tokens": null,
"min_new_tokens": null,
"seed": null,
"stop_tokens": null,
"temperature": 0,
"top_k": null,
"min_p": null,
"top_p": null,
"truncate_prompt": null,
"cache_implementation": null,
"response_format": null,
"skip_special_tokens": true
},
"system_prompt": null,
"cache_dir": "~/.cache/huggingface/lighteval",
"tokenizer": null,
"revision": "main",
"dtype": "bfloat16",
"tensor_parallel_size": 1,
"data_parallel_size": 1,
"pipeline_parallel_size": 1,
"gpu_memory_utilization": 0.9,
"enable_prefix_caching": null,
"max_model_length": null,
"quantization": null,
"load_format": null,
"swap_space": 4,
"seed": 1234,
"trust_remote_code": true,
"add_special_tokens": true,
"multichoice_continuations_start_space": true,
"pairwise_tokenization": false,
"max_num_seqs": 128,
"max_num_batched_tokens": 2048,
"subfolder": null,
"is_async": false,
"override_chat_template": null,
"enforce_eager": false
},
"model_name": "models/ministral-base"
},
"versions": {},
"results": {
"e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": {
"acc": 0.25109170305676853,
"acc_stderr": 0.020284878098207157,
"em_relaxed": 0.25327510917030566,
"em_relaxed_stderr": 0.020343162564319364
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
"acc": 0.19638242894056848,
"acc_stderr": 0.020220056896337255,
"em_relaxed": 0.19896640826873385,
"em_relaxed_stderr": 0.02031990120797851
},
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
"acc": 0.12337662337662338,
"acc_stderr": 0.01876954635295657,
"em_relaxed": 0.12337662337662338,
"em_relaxed_stderr": 0.01876954635295657
},
"scio.prompt1_mqa_single_correct_sk:mat|0": {
"acc": 0.23497267759562843,
"acc_stderr": 0.022192218701649325,
"em_relaxed": 0.2459016393442623,
"em_relaxed_stderr": 0.02253970761756048
},
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
"acc": 0.1981981981981982,
"acc_stderr": 0.026815582053277193,
"em_relaxed": 0.20270270270270271,
"em_relaxed_stderr": 0.027042309486278813
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
"acc": 0.20522388059701493,
"acc_stderr": 0.017460611985170203,
"em_relaxed": 0.21082089552238806,
"em_relaxed_stderr": 0.017634685741009386
},
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
"acc": 0.16666666666666666,
"acc_stderr": 0.02506903489984115,
"em_relaxed": 0.1891891891891892,
"em_relaxed_stderr": 0.02634582401273633
},
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
"acc": 0.1891891891891892,
"acc_stderr": 0.02438361375395457,
"em_relaxed": 0.1891891891891892,
"em_relaxed_stderr": 0.02438361375395457
},
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
"acc": 0.2644927536231884,
"acc_stderr": 0.026597083702337265,
"em_relaxed": 0.26811594202898553,
"em_relaxed_stderr": 0.02671259720934512
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
"acc": 0.2878787878787879,
"acc_stderr": 0.03225883512300992,
"em_relaxed": 0.2878787878787879,
"em_relaxed_stderr": 0.03225883512300992
},
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
"acc": 0.17,
"acc_stderr": 0.03775251680686371,
"em_relaxed": 0.17,
"em_relaxed_stderr": 0.03775251680686371
},
"slovak_tests.prompt3_qa_sk:maturita|0": {
"qem": 0.01327433628318584,
"qem_stderr": 0.007629803632081149,
"qf1": 0.0022123893805309734,
"qf1_stderr": 0.0015609152277667225,
"rouge1": 0.0013282561753440866,
"rouge1_stderr": 0.0009107295550233445,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0013282561753440866,
"rougeL_stderr": 0.0009107295550233445
},
"slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": {
"acc": 0.24423963133640553,
"acc_stderr": 0.029232990766796058,
"em_relaxed": 0.24423963133640553,
"em_relaxed_stderr": 0.029232990766796058
},
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
"acc": 0.25,
"acc_stderr": 0.033113308926626096,
"em_relaxed": 0.25,
"em_relaxed_stderr": 0.033113308926626096
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
"acc": 0.3939393939393939,
"acc_stderr": 0.04935824351078519,
"em_relaxed": 0.41414141414141414,
"em_relaxed_stderr": 0.04975740158605099
},
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
"acc": 0.16964285714285715,
"acc_stderr": 0.035623678500953895,
"em_relaxed": 0.1875,
"em_relaxed_stderr": 0.0370468111477387
},
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
"acc": 0.17682926829268292,
"acc_stderr": 0.02988327785748596,
"em_relaxed": 0.18902439024390244,
"em_relaxed_stderr": 0.030666839281449514
},
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
"acc": 0.2462686567164179,
"acc_stderr": 0.037358294208002164,
"em_relaxed": 0.2537313432835821,
"em_relaxed_stderr": 0.037731914327133735
},
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
"acc": 0.29,
"acc_stderr": 0.045604802157206845,
"em_relaxed": 0.3,
"em_relaxed_stderr": 0.046056618647183814
},
"e_tests.prompt2_mqa_single_correct_sk:geografia|0": {
"acc": 0.2857142857142857,
"acc_stderr": 0.06520506636966263,
"em_relaxed": 0.2857142857142857,
"em_relaxed_stderr": 0.06520506636966263
},
"e_tests.prompt2_mqa_single_correct_sk:chemia|0": {
"acc": 0.07142857142857142,
"acc_stderr": 0.034726602486028414,
"em_relaxed": 0.08928571428571429,
"em_relaxed_stderr": 0.038450387280282494
},
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
"acc": 0.1656441717791411,
"acc_stderr": 0.020621585415044594,
"em_relaxed": 0.17484662576687116,
"em_relaxed_stderr": 0.021069502803436863
},
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
"acc": 0.1875,
"acc_stderr": 0.0370468111477387,
"em_relaxed": 0.19642857142857142,
"em_relaxed_stderr": 0.03770970049347017
},
"movie_review.prompt2_sentiment_binary_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"e_tests.prompt1_mqa_single_correct_sk:biologia|0": {
"acc": 0.19540229885057472,
"acc_stderr": 0.0427567811097387,
"em_relaxed": 0.20689655172413793,
"em_relaxed_stderr": 0.04368097459950702
},
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
"acc": 0.21,
"acc_stderr": 0.040936018074033256,
"em_relaxed": 0.21,
"em_relaxed_stderr": 0.040936018074033256
},
"e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
"acc": 0.1565217391304348,
"acc_stderr": 0.034030776100725475,
"em_relaxed": 0.2,
"em_relaxed_stderr": 0.03746343246326772
},
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
"acc": 0.22,
"acc_stderr": 0.04163331998932268,
"em_relaxed": 0.22,
"em_relaxed_stderr": 0.04163331998932268
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
"acc": 0.22727272727272727,
"acc_stderr": 0.029857515673386393,
"em_relaxed": 0.23232323232323232,
"em_relaxed_stderr": 0.030088629490217483
},
"movie_review.prompt3_sentiment_binary_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slovak_tests.prompt2_qa_sk:maturita|0": {
"qem": 0.01327433628318584,
"qem_stderr": 0.007629803632081149,
"qf1": 0.002753195673549656,
"qf1_stderr": 0.0020208822149325673,
"rouge1": 0.0008509189925119129,
"rouge1_stderr": 0.0007009516038640078,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0008509189925119129,
"rougeL_stderr": 0.0007009516038640078
},
"e_tests.prompt1_qa_sk:slovensky-jazyk|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0025906735751295338,
"qf1_stderr": 0.0025906735751295403,
"rouge1": 5.842773836305639e-05,
"rouge1_stderr": 4.122343590294019e-05,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 5.842773836305639e-05,
"rougeL_stderr": 4.122343590294019e-05
},
"e_tests.prompt2_qa_sk:dejepis|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": {
"acc": 0.2205240174672489,
"acc_stderr": 0.019394170839568875,
"em_relaxed": 0.22270742358078602,
"em_relaxed_stderr": 0.019462629121054217
},
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
"acc": 0.23026315789473684,
"acc_stderr": 0.03426059424403165,
"em_relaxed": 0.23684210526315788,
"em_relaxed_stderr": 0.03459777606810535
},
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
"acc": 0.2702702702702703,
"acc_stderr": 0.02764841916772191,
"em_relaxed": 0.28185328185328185,
"em_relaxed_stderr": 0.028009688894113486
},
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
"acc": 0.21100917431192662,
"acc_stderr": 0.039262212380368526,
"em_relaxed": 0.22018348623853212,
"em_relaxed_stderr": 0.03987279758533731
},
"e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": {
"acc": 0.045454545454545456,
"acc_stderr": 0.031765283723714
},
"slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": {
"acc": 0.19815668202764977,
"acc_stderr": 0.02712203315362448,
"em_relaxed": 0.19815668202764977,
"em_relaxed_stderr": 0.02712203315362448
},
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
"acc": 0.2092426187419769,
"acc_stderr": 0.014583336673745648,
"em_relaxed": 0.21181001283697048,
"em_relaxed_stderr": 0.014648693814173547
},
"e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"e_tests.prompt2_mqa_single_correct_sk:biologia|0": {
"acc": 0.14942528735632185,
"acc_stderr": 0.03844315810763212,
"em_relaxed": 0.14942528735632185,
"em_relaxed_stderr": 0.03844315810763212
},
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
"acc": 0.23780487804878048,
"acc_stderr": 0.03334645408665337,
"em_relaxed": 0.24390243902439024,
"em_relaxed_stderr": 0.03363591048272823
},
"e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
"acc": 0.1232876712328767,
"acc_stderr": 0.022266919281488138,
"em_relaxed": 0.1232876712328767,
"em_relaxed_stderr": 0.022266919281488138
},
"e_tests.prompt1_qa_sk:geografia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"sklep.prompt3_nli_3_classes_sk:nli|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"scio.prompt3_mqa_single_correct_sk:vsp|0": {
"acc": 0.20388349514563106,
"acc_stderr": 0.014043631695334572,
"em_relaxed": 0.21723300970873785,
"em_relaxed_stderr": 0.014374053054620224
},
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
"acc": 0.2848101265822785,
"acc_stderr": 0.03601957344556447,
"em_relaxed": 0.2848101265822785,
"em_relaxed_stderr": 0.03601957344556447
},
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"arc.prompt1_mqa_single_correct_sk|0": {
"acc": 0.2728828058169376,
"acc_stderr": 0.013033734552785697,
"em_relaxed": 0.2728828058169376,
"em_relaxed_stderr": 0.013033734552785697
},
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
"acc": 0.18181818181818182,
"acc_stderr": 0.038961038961039,
"em_relaxed": 0.18181818181818182,
"em_relaxed_stderr": 0.038961038961039
},
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
"acc": 0.2616822429906542,
"acc_stderr": 0.0426929191572811,
"em_relaxed": 0.27102803738317754,
"em_relaxed_stderr": 0.04317273776566666
},
"belebele.prompt2_mqa_single_correct_sk|0": {
"acc": 0.2511111111111111,
"acc_stderr": 0.014463114105170805,
"em_relaxed": 0.25666666666666665,
"em_relaxed_stderr": 0.01456789134238002
},
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
"acc": 0.20922288642186165,
"acc_stderr": 0.0118915566619952,
"em_relaxed": 0.21178479931682323,
"em_relaxed_stderr": 0.011944744511660903
},
"senti_sk.prompt2_sentiment_3_class_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
"acc": 0.22018348623853212,
"acc_stderr": 0.03987279758533728,
"em_relaxed": 0.22018348623853212,
"em_relaxed_stderr": 0.03987279758533728
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
"acc": 0.13333333333333333,
"acc_stderr": 0.029365879728106854,
"em_relaxed": 0.13333333333333333,
"em_relaxed_stderr": 0.029365879728106854
},
"e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": {
"acc": 0.014285714285714285,
"acc_stderr": 0.010065123433632259
},
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slovakcopa.prompt3_copa_sk|0": {
"acc": 0.144,
"acc_stderr": 0.015716934945725767,
"em_relaxed": 0.144,
"em_relaxed_stderr": 0.015716934945725767
},
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
"acc": 0.2852664576802508,
"acc_stderr": 0.025321176274679096,
"em_relaxed": 0.2852664576802508,
"em_relaxed_stderr": 0.025321176274679096
},
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
"acc": 0.1651376146788991,
"acc_stderr": 0.0357288117600854,
"em_relaxed": 0.1651376146788991,
"em_relaxed_stderr": 0.0357288117600854
},
"e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
"acc": 0.3656716417910448,
"acc_stderr": 0.041761576413228745,
"em_relaxed": 0.373134328358209,
"em_relaxed_stderr": 0.041936678130523646
},
"slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": {
"acc": 0.21683673469387754,
"acc_stderr": 0.02084031144263498,
"em_relaxed": 0.2193877551020408,
"em_relaxed_stderr": 0.020928374259338522
},
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
"acc": 0.24,
"acc_stderr": 0.021380899352993928,
"em_relaxed": 0.2425,
"em_relaxed_stderr": 0.021456591955524264
},
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
"acc": 0.30638297872340425,
"acc_stderr": 0.03013590647851756,
"em_relaxed": 0.3148936170212766,
"em_relaxed_stderr": 0.03036358219723817
},
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
"acc": 0.22402597402597402,
"acc_stderr": 0.023795979800861743,
"em_relaxed": 0.22402597402597402,
"em_relaxed_stderr": 0.023795979800861743
},
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
"acc": 0.2911392405063291,
"acc_stderr": 0.03625609398754825,
"em_relaxed": 0.2974683544303797,
"em_relaxed_stderr": 0.03648408889243667
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
"acc": 0.19689119170984457,
"acc_stderr": 0.028697873971860688,
"em_relaxed": 0.19689119170984457,
"em_relaxed_stderr": 0.028697873971860688
},
"scio.prompt3_mqa_single_correct_sk:mat|0": {
"acc": 0.20765027322404372,
"acc_stderr": 0.021231382642567846,
"em_relaxed": 0.21584699453551912,
"em_relaxed_stderr": 0.02153411244027524
},
"e_tests.prompt2_qa_sk:geografia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
"acc": 0.21100917431192662,
"acc_stderr": 0.03926221238036852,
"em_relaxed": 0.21100917431192662,
"em_relaxed_stderr": 0.03926221238036852
},
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
"acc": 0.17094017094017094,
"acc_stderr": 0.02466249684520983,
"em_relaxed": 0.17094017094017094,
"em_relaxed_stderr": 0.02466249684520983
},
"scio.prompt1_mqa_single_correct_sk:bio|0": {
"acc": 0.2743362831858407,
"acc_stderr": 0.024268949584718702,
"em_relaxed": 0.28023598820059,
"em_relaxed_stderr": 0.024428605014452955
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
"acc": 0.24352331606217617,
"acc_stderr": 0.03097543638684542,
"em_relaxed": 0.24870466321243523,
"em_relaxed_stderr": 0.0311958408777003
},
"sklep.prompt3_qa_sk:question-answering|0": {
"qem": 0.0002087029114056141,
"qem_stderr": 0.0001475675430580104,
"qf1": 0.006093522810123945,
"qf1_stderr": 0.00032767921496533014
},
"e_tests.prompt3_qa_sk:biologia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0011204481792717084,
"rouge1_stderr": 0.0011204481792717086,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0011204481792717084,
"rougeL_stderr": 0.0011204481792717086
},
"slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": {
"acc": 0.13333333333333333,
"acc_stderr": 0.03333333333333327,
"em_relaxed": 0.13333333333333333,
"em_relaxed_stderr": 0.03333333333333327
},
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
"acc": 0.23134328358208955,
"acc_stderr": 0.03656527469759901,
"em_relaxed": 0.23134328358208955,
"em_relaxed_stderr": 0.03656527469759901
},
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
"acc": 0.2182741116751269,
"acc_stderr": 0.029505319811543015,
"em_relaxed": 0.2182741116751269,
"em_relaxed_stderr": 0.029505319811543015
},
"e_tests.prompt2_mqa_single_correct_sk:matematika|0": {
"acc": 0.10256410256410256,
"acc_stderr": 0.04921614717627658,
"em_relaxed": 0.3076923076923077,
"em_relaxed_stderr": 0.074871425137275
},
"scio.prompt1_mqa_single_correct_sk:aj|0": {
"acc": 0.6,
"acc_stderr": 0.16329931618554522,
"em_relaxed": 0.6,
"em_relaxed_stderr": 0.16329931618554522
},
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
"acc": 0.22393822393822393,
"acc_stderr": 0.02595388693967799,
"em_relaxed": 0.23166023166023167,
"em_relaxed_stderr": 0.026265915716011466
},
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
"acc": 0.2,
"acc_stderr": 0.04020151261036846,
"em_relaxed": 0.2,
"em_relaxed_stderr": 0.04020151261036846
},
"financial_exams.prompt1_completion_sk|0": {
"acc": 0.053973013493253376,
"acc_stderr": 0.006189064679835392,
"em_relaxed": 0.05697151424287856,
"em_relaxed_stderr": 0.006348574383669881
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
"acc": 0.28110599078341014,
"acc_stderr": 0.03058726229145915,
"em_relaxed": 0.2903225806451613,
"em_relaxed_stderr": 0.030884745411494743
},
"slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0012698412698412698,
"rouge1_stderr": 0.00126984126984127,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0012698412698412698,
"rougeL_stderr": 0.00126984126984127
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
"acc": 0.22807017543859648,
"acc_stderr": 0.056069817847611766,
"em_relaxed": 0.22807017543859648,
"em_relaxed_stderr": 0.056069817847611766
},
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
"acc": 0.2753623188405797,
"acc_stderr": 0.02693682252998795,
"em_relaxed": 0.27898550724637683,
"em_relaxed_stderr": 0.027045590734089176
},
"ifeval_sk.ifeval|0": {
"inst_level_loose_acc": 0.17312348668280872,
"inst_level_loose_acc_stderr": 0.0004210039776560394,
"inst_level_strict_acc": 0.17312348668280872,
"inst_level_strict_acc_stderr": 0.0004210039776560394,
"prompt_level_loose_acc": 0.0782122905027933,
"prompt_level_loose_acc_stderr": 0.011597665111537078,
"prompt_level_strict_acc": 0.0782122905027933,
"prompt_level_strict_acc_stderr": 0.011597665111537078
},
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
"acc": 0.2826086956521739,
"acc_stderr": 0.027152165210427823,
"em_relaxed": 0.286231884057971,
"em_relaxed_stderr": 0.02725657169175206
},
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
"acc": 0.2314261315115286,
"acc_stderr": 0.012329801197790603,
"em_relaxed": 0.23825789923142612,
"em_relaxed_stderr": 0.012454741062931133
},
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
"acc": 0.1834862385321101,
"acc_stderr": 0.03724531275523975,
"em_relaxed": 0.1834862385321101,
"em_relaxed_stderr": 0.03724531275523975
},
"movie_review.prompt1_sentiment_binary_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slovak_tests.prompt1_qa_sk:monitor9|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
"acc": 0.2416555407209613,
"acc_stderr": 0.01565239642514941,
"em_relaxed": 0.2416555407209613,
"em_relaxed_stderr": 0.01565239642514941
},
"slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": {
"acc": 0.15718157181571815,
"acc_stderr": 0.018973345526861658,
"em_relaxed": 0.16260162601626016,
"em_relaxed_stderr": 0.019235549151947248
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
"acc": 0.22897196261682243,
"acc_stderr": 0.028789653442089263,
"em_relaxed": 0.24766355140186916,
"em_relaxed_stderr": 0.02957653529316449
},
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
"acc": 0.17391304347826086,
"acc_stderr": 0.03549985134468377,
"em_relaxed": 0.20869565217391303,
"em_relaxed_stderr": 0.03806063686277675
},
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
"acc": 0.21983914209115282,
"acc_stderr": 0.02147203067557281,
"em_relaxed": 0.21983914209115282,
"em_relaxed_stderr": 0.02147203067557281
},
"toxic_sk.prompt1_toxicity_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"driving_test.prompt2_mqa_single_correct_sk|0": {
"acc": 0.09985734664764621,
"acc_stderr": 0.011331741573886145,
"em_relaxed": 0.10271041369472182,
"em_relaxed_stderr": 0.011474255943096628
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
"acc": 0.32323232323232326,
"acc_stderr": 0.04724590344515123,
"em_relaxed": 0.3333333333333333,
"em_relaxed_stderr": 0.04761904761904759
},
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
"acc": 0.19736842105263158,
"acc_stderr": 0.03238981601699397,
"em_relaxed": 0.21052631578947367,
"em_relaxed_stderr": 0.033176727875331574
},
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
"acc": 0.2803030303030303,
"acc_stderr": 0.027695605168511307,
"em_relaxed": 0.2878787878787879,
"em_relaxed_stderr": 0.027919261062611087
},
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
"acc": 0.2322801024765158,
"acc_stderr": 0.012345664537818761,
"em_relaxed": 0.23654995730145176,
"em_relaxed_stderr": 0.012423924913578318
},
"sklep.prompt2_nli_3_classes_sk:nli|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
"acc": 0.2119815668202765,
"acc_stderr": 0.02780931949344348,
"em_relaxed": 0.22580645161290322,
"em_relaxed_stderr": 0.028448938828651484
},
"sklep.prompt1_nli_binary_sk:rte|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"e_tests.prompt3_qa_sk:dejepis|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
"acc": 0.22,
"acc_stderr": 0.0416333199893227,
"em_relaxed": 0.24,
"em_relaxed_stderr": 0.04292346959909283
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
"acc": 0.2631578947368421,
"acc_stderr": 0.05884389414473131,
"em_relaxed": 0.2807017543859649,
"em_relaxed_stderr": 0.06004585739704729
},
"slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": {
"acc": 0.13520408163265307,
"acc_stderr": 0.017292721140889967,
"em_relaxed": 0.1377551020408163,
"em_relaxed_stderr": 0.01742933367587574
},
"scio.prompt3_mqa_single_correct_sk:che|0": {
"acc": 0.22764227642276422,
"acc_stderr": 0.026788763286110986,
"em_relaxed": 0.23577235772357724,
"em_relaxed_stderr": 0.027119068752977968
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
"acc": 0.23333333333333334,
"acc_stderr": 0.03464962582372412,
"em_relaxed": 0.23333333333333334,
"em_relaxed_stderr": 0.03464962582372412
},
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
"acc": 0.1919191919191919,
"acc_stderr": 0.03978080447933685,
"em_relaxed": 0.1919191919191919,
"em_relaxed_stderr": 0.03978080447933685
},
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
"acc": 0.16071428571428573,
"acc_stderr": 0.034859460964757394,
"em_relaxed": 0.16071428571428573,
"em_relaxed_stderr": 0.034859460964757394
},
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
"acc": 0.2535211267605634,
"acc_stderr": 0.03663585085277138,
"em_relaxed": 0.2605633802816901,
"em_relaxed_stderr": 0.03696558732615873
},
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
"acc": 0.22442244224422442,
"acc_stderr": 0.024007242951044432,
"em_relaxed": 0.25412541254125415,
"em_relaxed_stderr": 0.02505263910183634
},
"slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.001330671989354624,
"rouge1_stderr": 0.0013306719893546237,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.001330671989354624,
"rougeL_stderr": 0.0013306719893546237
},
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
"acc": 0.18711656441717792,
"acc_stderr": 0.021633588964875924,
"em_relaxed": 0.19325153374233128,
"em_relaxed_stderr": 0.02190225688747218
},
"e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": {
"acc": 0.21615720524017468,
"acc_stderr": 0.0192548989397317,
"em_relaxed": 0.2183406113537118,
"em_relaxed_stderr": 0.01932493025894029
},
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
"acc": 0.14,
"acc_stderr": 0.0348735088019777,
"em_relaxed": 0.16,
"em_relaxed_stderr": 0.03684529491774708
},
"e_tests.prompt1_mqa_single_correct_sk:geografia|0": {
"acc": 0.20408163265306123,
"acc_stderr": 0.058172215566282555,
"em_relaxed": 0.20408163265306123,
"em_relaxed_stderr": 0.058172215566282555
},
"e_tests.prompt2_qa_sk:fyzika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"multi_wiki_qa.prompt3_qa_sk|0": {
"qem": 0.0006003602161296778,
"qem_stderr": 0.00034654874631857635,
"qf1": 0.005765729879562343,
"qf1_stderr": 0.0005521722468476828
},
"e_tests.prompt1_qa_sk:fyzika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
"acc": 0.19172932330827067,
"acc_stderr": 0.024182419147367247,
"em_relaxed": 0.20676691729323307,
"em_relaxed_stderr": 0.024878145232561027
},
"e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": {
"acc": 0.15942028985507245,
"acc_stderr": 0.04439221265797129,
"em_relaxed": 0.15942028985507245,
"em_relaxed_stderr": 0.04439221265797129
},
"e_tests.prompt2_mqa_single_correct_sk:dejepis|0": {
"acc": 0.20833333333333334,
"acc_stderr": 0.059238169759886904,
"em_relaxed": 0.20833333333333334,
"em_relaxed_stderr": 0.059238169759886904
},
"scio.prompt1_mqa_single_correct_sk:vsp|0": {
"acc": 0.24150485436893204,
"acc_stderr": 0.01491898981873877,
"em_relaxed": 0.25849514563106796,
"em_relaxed_stderr": 0.015261010021381633
},
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
"acc": 0.16,
"acc_stderr": 0.03684529491774709,
"em_relaxed": 0.17,
"em_relaxed_stderr": 0.0377525168068637
},
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
"acc": 0.3050847457627119,
"acc_stderr": 0.04256799926288002,
"em_relaxed": 0.3050847457627119,
"em_relaxed_stderr": 0.04256799926288002
},
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
"acc": 0.288135593220339,
"acc_stderr": 0.04187011593049808,
"em_relaxed": 0.2966101694915254,
"em_relaxed_stderr": 0.04222776832233627
},
"slovakcopa.prompt2_copa_sk|0": {
"acc": 0.006,
"acc_stderr": 0.003457152557758384,
"em_relaxed": 0.006,
"em_relaxed_stderr": 0.003457152557758384
},
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
"acc": 0.2116564417177914,
"acc_stderr": 0.022658534978587233,
"em_relaxed": 0.22085889570552147,
"em_relaxed_stderr": 0.023010381863796215
},
"e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
"acc": 0.24,
"acc_stderr": 0.042923469599092816,
"em_relaxed": 0.25,
"em_relaxed_stderr": 0.04351941398892446
},
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
"acc": 0.2807017543859649,
"acc_stderr": 0.034462962170884265,
"em_relaxed": 0.2807017543859649,
"em_relaxed_stderr": 0.034462962170884265
},
"e_tests.prompt1_qa_sk:biologia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
"acc": 0.15,
"acc_stderr": 0.03588702812826371,
"em_relaxed": 0.15,
"em_relaxed_stderr": 0.03588702812826371
},
"e_tests.prompt3_mqa_single_correct_sk:chemia|0": {
"acc": 0.07142857142857142,
"acc_stderr": 0.03472660248602842,
"em_relaxed": 0.08928571428571429,
"em_relaxed_stderr": 0.038450387280282494
},
"e_tests.prompt1_mqa_single_correct_sk:matematika|0": {
"acc": 0.15384615384615385,
"acc_stderr": 0.05852968540405249,
"em_relaxed": 0.358974358974359,
"em_relaxed_stderr": 0.07781756136754926
},
"e_tests.prompt3_qa_sk:geografia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": {
"acc": 0.045454545454545456,
"acc_stderr": 0.031765283723714
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
"acc": 0.2383177570093458,
"acc_stderr": 0.029192770642101563,
"em_relaxed": 0.2570093457943925,
"em_relaxed_stderr": 0.029941691533244646
},
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
"acc": 0.24932975871313673,
"acc_stderr": 0.022430563714050267,
"em_relaxed": 0.24932975871313673,
"em_relaxed_stderr": 0.022430563714050267
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
"acc": 0.24468085106382978,
"acc_stderr": 0.025645553622266722,
"em_relaxed": 0.25177304964539005,
"em_relaxed_stderr": 0.025892151156709405
},
"e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
"acc": 0.22535211267605634,
"acc_stderr": 0.035186286213623755,
"em_relaxed": 0.22535211267605634,
"em_relaxed_stderr": 0.035186286213623755
},
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
"acc": 0.296551724137931,
"acc_stderr": 0.03806142687309993,
"em_relaxed": 0.3103448275862069,
"em_relaxed_stderr": 0.03855289616378948
},
"e_tests.prompt1_mqa_single_correct_sk:chemia|0": {
"acc": 0.10714285714285714,
"acc_stderr": 0.0417053005800816,
"em_relaxed": 0.10714285714285714,
"em_relaxed_stderr": 0.0417053005800816
},
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
"acc": 0.25671140939597314,
"acc_stderr": 0.01790782476183757,
"em_relaxed": 0.25838926174496646,
"em_relaxed_stderr": 0.017945962358551722
},
"e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": {
"acc": 0.045454545454545456,
"acc_stderr": 0.031765283723714
},
"e_tests.prompt3_qa_sk:obcianska-nauka|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
"acc": 0.2,
"acc_stderr": 0.04020151261036843,
"em_relaxed": 0.22,
"em_relaxed_stderr": 0.041633319989322716
},
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
"acc": 0.2018348623853211,
"acc_stderr": 0.03862180715692686,
"em_relaxed": 0.2018348623853211,
"em_relaxed_stderr": 0.03862180715692686
},
"e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": {
"acc": 0.14492753623188406,
"acc_stderr": 0.0426896366873121,
"em_relaxed": 0.14492753623188406,
"em_relaxed_stderr": 0.0426896366873121
},
"e_tests.prompt3_mqa_single_correct_sk:fyzika|0": {
"acc": 0.18181818181818182,
"acc_stderr": 0.08416546361568647,
"em_relaxed": 0.2727272727272727,
"em_relaxed_stderr": 0.0971859061499725
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
"acc": 0.16917293233082706,
"acc_stderr": 0.023030205866522328,
"em_relaxed": 0.17669172932330826,
"em_relaxed_stderr": 0.023429683004809066
},
"e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": {
"acc": 0.034482758620689655,
"acc_stderr": 0.034482758620689676
},
"e_tests.prompt3_qa_sk:chemia|0": {
"qem": 0.018518518518518517,
"qem_stderr": 0.018518518518518517,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0006660882976672449,
"rouge1_stderr": 0.0004499203448417917,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0006660882976672449,
"rougeL_stderr": 0.0004499203448417917
},
"e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": {
"acc": 0.03571428571428571,
"acc_stderr": 0.02502318034804895
},
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
"acc": 0.1826086956521739,
"acc_stderr": 0.03618456100987082,
"em_relaxed": 0.21739130434782608,
"em_relaxed_stderr": 0.038631448549506
},
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
"acc": 0.21379310344827587,
"acc_stderr": 0.034165204477475494,
"em_relaxed": 0.2206896551724138,
"em_relaxed_stderr": 0.03455930201924811
},
"e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": {
"acc": 0.014285714285714285,
"acc_stderr": 0.010065123433632259
},
"e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
"acc": 0.22,
"acc_stderr": 0.041633319989322695,
"em_relaxed": 0.24,
"em_relaxed_stderr": 0.042923469599092816
},
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
"acc": 0.24786324786324787,
"acc_stderr": 0.028286324075564397,
"em_relaxed": 0.25213675213675213,
"em_relaxed_stderr": 0.02844796547623102
},
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
"acc": 0.26900584795321636,
"acc_stderr": 0.0340105262010409,
"em_relaxed": 0.27485380116959063,
"em_relaxed_stderr": 0.034240429246915824
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
"acc": 0.21717171717171718,
"acc_stderr": 0.02937661648494564,
"em_relaxed": 0.21717171717171718,
"em_relaxed_stderr": 0.02937661648494564
},
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
"acc": 0.26702269692923897,
"acc_stderr": 0.01617590372442817,
"em_relaxed": 0.26702269692923897,
"em_relaxed_stderr": 0.01617590372442817
},
"e_tests.prompt2_qa_sk:informatika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.010073260073260074,
"qf1_stderr": 0.007341600122632042,
"rouge1": 0.0012475950143262986,
"rouge1_stderr": 0.000686304468853021,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0012475950143262986,
"rougeL_stderr": 0.000686304468853021
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
"acc": 0.2474747474747475,
"acc_stderr": 0.030746300742124498,
"em_relaxed": 0.25252525252525254,
"em_relaxed_stderr": 0.030954055470365897
},
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
"acc": 0.24242424242424243,
"acc_stderr": 0.04329004329004326,
"em_relaxed": 0.26262626262626265,
"em_relaxed_stderr": 0.04445287676983944
},
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
"acc": 0.16964285714285715,
"acc_stderr": 0.0356236785009539,
"em_relaxed": 0.19642857142857142,
"em_relaxed_stderr": 0.03770970049347018
},
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
"acc": 0.22560975609756098,
"acc_stderr": 0.03273897454566342,
"em_relaxed": 0.22560975609756098,
"em_relaxed_stderr": 0.03273897454566342
},
"slovakcopa.prompt1_copa_sk|0": {
"acc": 0.01,
"acc_stderr": 0.0044541754970721,
"em_relaxed": 0.01,
"em_relaxed_stderr": 0.0044541754970721
},
"scio.prompt3_mqa_single_correct_sk:bio|0": {
"acc": 0.3008849557522124,
"acc_stderr": 0.024946878423156852,
"em_relaxed": 0.30973451327433627,
"em_relaxed_stderr": 0.025150378763664
},
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
"acc": 0.205607476635514,
"acc_stderr": 0.03925401580070484,
"em_relaxed": 0.21495327102803738,
"em_relaxed_stderr": 0.03989944463395407
},
"multi_wiki_qa.prompt2_qa_sk|0": {
"qem": 0.0006003602161296778,
"qem_stderr": 0.0003465487463185779,
"qf1": 0.005417769336679904,
"qf1_stderr": 0.0005509525930025259
},
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
"acc": 0.21568627450980393,
"acc_stderr": 0.04092563958237654,
"em_relaxed": 0.22549019607843138,
"em_relaxed_stderr": 0.04158307533083286
},
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
"acc": 0.20425531914893616,
"acc_stderr": 0.026355158413349424,
"em_relaxed": 0.20425531914893616,
"em_relaxed_stderr": 0.026355158413349424
},
"e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": {
"acc": 0.18840579710144928,
"acc_stderr": 0.047420064740574204,
"em_relaxed": 0.18840579710144928,
"em_relaxed_stderr": 0.047420064740574204
},
"e_tests.prompt3_mqa_single_correct_sk:dejepis|0": {
"acc": 0.20833333333333334,
"acc_stderr": 0.059238169759886904,
"em_relaxed": 0.20833333333333334,
"em_relaxed_stderr": 0.059238169759886904
},
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
"acc": 0.19122257053291536,
"acc_stderr": 0.022053144718757862,
"em_relaxed": 0.19122257053291536,
"em_relaxed_stderr": 0.022053144718757862
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
"acc": 0.23333333333333334,
"acc_stderr": 0.03464962582372412,
"em_relaxed": 0.23333333333333334,
"em_relaxed_stderr": 0.03464962582372412
},
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"driving_test.prompt1_mqa_single_correct_sk|0": {
"acc": 0.16405135520684735,
"acc_stderr": 0.01399686230459443,
"em_relaxed": 0.16833095577746077,
"em_relaxed_stderr": 0.014141915666582279
},
"e_tests.prompt3_mqa_single_correct_sk:informatika|0": {
"acc": 0.256,
"acc_stderr": 0.03919183588453089,
"em_relaxed": 0.264,
"em_relaxed_stderr": 0.03958494337416042
},
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
"acc": 0.18959731543624161,
"acc_stderr": 0.01606971018943585,
"em_relaxed": 0.19463087248322147,
"em_relaxed_stderr": 0.0162309847870997
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
"acc": 0.22429906542056074,
"acc_stderr": 0.028580583273338626,
"em_relaxed": 0.2383177570093458,
"em_relaxed_stderr": 0.029192770642101563
},
"e_tests.prompt1_mqa_single_correct_sk:fyzika|0": {
"acc": 0.13636363636363635,
"acc_stderr": 0.07488677009526491,
"em_relaxed": 0.22727272727272727,
"em_relaxed_stderr": 0.09144861547306321
},
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
"acc": 0.26744186046511625,
"acc_stderr": 0.03384836428157859,
"em_relaxed": 0.27906976744186046,
"em_relaxed_stderr": 0.03430085607014882
},
"slovak_tests.prompt3_qa_sk:monitor9|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
"acc": 0.136986301369863,
"acc_stderr": 0.023287301596102452,
"em_relaxed": 0.136986301369863,
"em_relaxed_stderr": 0.023287301596102452
},
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
"acc": 0.2025,
"acc_stderr": 0.0201183061066658,
"em_relaxed": 0.205,
"em_relaxed_stderr": 0.02021035988339997
},
"sklep.prompt2_qa_sk:question-answering|0": {
"qem": 0.0002087029114056141,
"qem_stderr": 0.0001475675430580104,
"qf1": 0.006743145308527306,
"qf1_stderr": 0.00039734127192685853
},
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
"acc": 0.04884318766066838,
"acc_stderr": 0.00773245768018304,
"em_relaxed": 0.08354755784061697,
"em_relaxed_stderr": 0.009926847196525902
},
"senti_sk.prompt3_sentiment_binary_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"scio.prompt2_mqa_single_correct_sk:vsp|0": {
"acc": 0.23786407766990292,
"acc_stderr": 0.014841600192116697,
"em_relaxed": 0.2512135922330097,
"em_relaxed_stderr": 0.015118218765838777
},
"e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"e_tests.prompt3_qa_sk:slovensky-jazyk|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0025906735751295338,
"qf1_stderr": 0.0025906735751295403,
"rouge1": 0.00035708568846413394,
"rouge1_stderr": 0.00020841637674444064,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.00035708568846413394,
"rougeL_stderr": 0.00020841637674444064
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
"acc": 0.24561403508771928,
"acc_stderr": 0.05752139056405265,
"em_relaxed": 0.2631578947368421,
"em_relaxed_stderr": 0.05884389414473131
},
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
"acc": 0.16595744680851063,
"acc_stderr": 0.02432117475103868,
"em_relaxed": 0.16595744680851063,
"em_relaxed_stderr": 0.02432117475103868
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
"acc": 0.2126865671641791,
"acc_stderr": 0.01769159425053082,
"em_relaxed": 0.21641791044776118,
"em_relaxed_stderr": 0.01780376934180082
},
"senti_sk.prompt2_sentiment_binary_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"scio.prompt1_mqa_single_correct_sk:che|0": {
"acc": 0.25609756097560976,
"acc_stderr": 0.027885450835842325,
"em_relaxed": 0.25609756097560976,
"em_relaxed_stderr": 0.027885450835842325
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
"acc": 0.24444444444444444,
"acc_stderr": 0.03212157057535212,
"em_relaxed": 0.25,
"em_relaxed_stderr": 0.032364888900157734
},
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
"acc": 0.14285714285714285,
"acc_stderr": 0.03321361106966269,
"em_relaxed": 0.16071428571428573,
"em_relaxed_stderr": 0.0348594609647574
},
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
"acc": 0.21212121212121213,
"acc_stderr": 0.04129606932540889,
"em_relaxed": 0.24242424242424243,
"em_relaxed_stderr": 0.04329004329004327
},
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
"acc": 0.26959247648902823,
"acc_stderr": 0.024884157592464488,
"em_relaxed": 0.26959247648902823,
"em_relaxed_stderr": 0.024884157592464488
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
"acc": 0.2538860103626943,
"acc_stderr": 0.03141024780565318,
"em_relaxed": 0.2538860103626943,
"em_relaxed_stderr": 0.03141024780565318
},
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
"acc": 0.21212121212121213,
"acc_stderr": 0.04129606932540889,
"em_relaxed": 0.23232323232323232,
"em_relaxed_stderr": 0.042660160170546885
},
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
"acc": 0.042416452442159386,
"acc_stderr": 0.0072301115841424965,
"em_relaxed": 0.07840616966580977,
"em_relaxed_stderr": 0.009643493430258573
},
"belebele.prompt3_mqa_single_correct_sk|0": {
"acc": 0.24666666666666667,
"acc_stderr": 0.014377023375409347,
"em_relaxed": 0.2544444444444444,
"em_relaxed_stderr": 0.014526354751055178
},
"arc.prompt3_mqa_single_correct_sk|0": {
"acc": 0.22840034217279725,
"acc_stderr": 0.012283523666042804,
"em_relaxed": 0.22840034217279725,
"em_relaxed_stderr": 0.012283523666042804
},
"financial_exams.prompt2_completion_sk|0": {
"acc": 0.07871064467766117,
"acc_stderr": 0.00737563958812383,
"em_relaxed": 0.07946026986506746,
"em_relaxed_stderr": 0.007407662917614668
},
"e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
"acc": 0.23049645390070922,
"acc_stderr": 0.025123739226872395,
"em_relaxed": 0.24113475177304963,
"em_relaxed_stderr": 0.02551873104953778
},
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
"acc": 0.24,
"acc_stderr": 0.042923469599092816,
"em_relaxed": 0.24,
"em_relaxed_stderr": 0.042923469599092816
},
"e_tests.prompt3_mqa_single_correct_sk:biologia|0": {
"acc": 0.10344827586206896,
"acc_stderr": 0.03283976780985287,
"em_relaxed": 0.11494252873563218,
"em_relaxed_stderr": 0.03439354046699091
},
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
"acc": 0.24299065420560748,
"acc_stderr": 0.041657429989652724,
"em_relaxed": 0.24299065420560748,
"em_relaxed_stderr": 0.041657429989652724
},
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
"acc": 0.2191780821917808,
"acc_stderr": 0.028018592326462144,
"em_relaxed": 0.2191780821917808,
"em_relaxed_stderr": 0.028018592326462144
},
"slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": {
"acc": 0.18157181571815717,
"acc_stderr": 0.020095119692986566,
"em_relaxed": 0.18157181571815717,
"em_relaxed_stderr": 0.020095119692986566
},
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
"acc": 0.22549019607843138,
"acc_stderr": 0.04158307533083286,
"em_relaxed": 0.23529411764705882,
"em_relaxed_stderr": 0.04220773659171455
},
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
"acc": 0.20535714285714285,
"acc_stderr": 0.03834241021419071,
"em_relaxed": 0.22321428571428573,
"em_relaxed_stderr": 0.039523019677025116
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
"acc": 0.1921641791044776,
"acc_stderr": 0.01703416618213854,
"em_relaxed": 0.1958955223880597,
"em_relaxed_stderr": 0.01715898536836359
},
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
"acc": 0.23529411764705882,
"acc_stderr": 0.04220773659171455,
"em_relaxed": 0.24509803921568626,
"em_relaxed_stderr": 0.042801058373643945
},
"e_tests.prompt1_mqa_single_correct_sk:dejepis|0": {
"acc": 0.22916666666666666,
"acc_stderr": 0.06130657275434794,
"em_relaxed": 0.22916666666666666,
"em_relaxed_stderr": 0.06130657275434794
},
"scio.prompt2_mqa_single_correct_sk:bio|0": {
"acc": 0.2861356932153392,
"acc_stderr": 0.02458303513041921,
"em_relaxed": 0.2949852507374631,
"em_relaxed_stderr": 0.024805095426679618
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
"acc": 0.23255813953488372,
"acc_stderr": 0.021502792400356754,
"em_relaxed": 0.24031007751937986,
"em_relaxed_stderr": 0.02174755921996615
},
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
"acc": 0.2537878787878788,
"acc_stderr": 0.02683420376859365,
"em_relaxed": 0.25757575757575757,
"em_relaxed_stderr": 0.026965016533893105
},
"toxic_sk.prompt3_toxicity_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
"acc": 0.17777777777777778,
"acc_stderr": 0.033027898599017176,
"em_relaxed": 0.17777777777777778,
"em_relaxed_stderr": 0.033027898599017176
},
"driving_test.prompt3_mqa_single_correct_sk|0": {
"acc": 0.14835948644793154,
"acc_stderr": 0.013434972977209091,
"em_relaxed": 0.15121255349500715,
"em_relaxed_stderr": 0.01354080168340582
},
"slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": {
"acc": 0.1760204081632653,
"acc_stderr": 0.019259781722629446,
"em_relaxed": 0.1760204081632653,
"em_relaxed_stderr": 0.019259781722629446
},
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
"acc": 0.19,
"acc_stderr": 0.019639610121239284,
"em_relaxed": 0.1925,
"em_relaxed_stderr": 0.01973786547054499
},
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
"acc": 0.12337662337662338,
"acc_stderr": 0.018769546352956558,
"em_relaxed": 0.12337662337662338,
"em_relaxed_stderr": 0.018769546352956558
},
"sklep.prompt1_qa_sk:question-answering|0": {
"qem": 0.0004174058228112282,
"qem_stderr": 0.00020867023775979663,
"qf1": 0.008276210006991061,
"qf1_stderr": 0.00042241089189288785
},
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
"acc": 0.2,
"acc_stderr": 0.04020151261036846,
"em_relaxed": 0.22,
"em_relaxed_stderr": 0.041633319989322674
},
"skj1.prompt1_skj1_sk|0": {
"acc": 0.41,
"acc_stderr": 0.04943110704237102,
"em_relaxed": 0.44,
"em_relaxed_stderr": 0.049888765156985884
},
"e_tests.prompt2_qa_sk:slovensky-jazyk|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0025906735751295338,
"qf1_stderr": 0.0025906735751295403,
"rouge1": 0.00011405864075384238,
"rouge1_stderr": 8.908193183205394e-05,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.00011405864075384238,
"rougeL_stderr": 8.908193183205394e-05
},
"belebele.prompt1_mqa_single_correct_sk|0": {
"acc": 0.2633333333333333,
"acc_stderr": 0.014689553047342515,
"em_relaxed": 0.2722222222222222,
"em_relaxed_stderr": 0.014845038794433947
},
"e_tests.prompt2_qa_sk:chemia|0": {
"qem": 0.018518518518518517,
"qem_stderr": 0.018518518518518517,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 3.292181069958847e-05,
"rouge1_stderr": 3.292181069958849e-05,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 3.292181069958847e-05,
"rougeL_stderr": 3.292181069958849e-05
},
"sklep.prompt2_nli_binary_sk:rte|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"multi_wiki_qa.prompt1_qa_sk|0": {
"qem": 0.00040024014408645187,
"qem_stderr": 0.00028298419465801113,
"qf1": 0.004608242571473309,
"qf1_stderr": 0.00046536745363106924
},
"slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": {
"acc": 0.21904761904761905,
"acc_stderr": 0.040556911537178254,
"em_relaxed": 0.21904761904761905,
"em_relaxed_stderr": 0.040556911537178254
},
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
"acc": 0.17,
"acc_stderr": 0.03775251680686371,
"em_relaxed": 0.17,
"em_relaxed_stderr": 0.03775251680686371
},
"skj1.prompt3_skj1_sk|0": {
"acc": 0.75,
"acc_stderr": 0.04351941398892446,
"em_relaxed": 0.75,
"em_relaxed_stderr": 0.04351941398892446
},
"slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0021574973031283713,
"rouge1_stderr": 0.0021574973031283713,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0010787486515641857,
"rougeL_stderr": 0.0010787486515641857
},
"e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": {
"acc": 0.034482758620689655,
"acc_stderr": 0.034482758620689676
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
"acc": 0.22777777777777777,
"acc_stderr": 0.03134731210385172,
"em_relaxed": 0.23333333333333334,
"em_relaxed_stderr": 0.031612960436120094
},
"scio.prompt2_mqa_single_correct_sk:mat|0": {
"acc": 0.21311475409836064,
"acc_stderr": 0.021434631795998253,
"em_relaxed": 0.22131147540983606,
"em_relaxed_stderr": 0.021728884019137824
},
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
"acc": 0.20930232558139536,
"acc_stderr": 0.03110958390976464,
"em_relaxed": 0.21511627906976744,
"em_relaxed_stderr": 0.03142253684735939
},
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
"acc": 0.18666666666666668,
"acc_stderr": 0.03192085664169663,
"em_relaxed": 0.19333333333333333,
"em_relaxed_stderr": 0.03235245865324252
},
"scio.prompt2_mqa_single_correct_sk:che|0": {
"acc": 0.18699186991869918,
"acc_stderr": 0.024910116828719235,
"em_relaxed": 0.1910569105691057,
"em_relaxed_stderr": 0.025116395774587044
},
"skj1.prompt2_skj1_sk|0": {
"acc": 0.63,
"acc_stderr": 0.04852365870939099,
"em_relaxed": 0.64,
"em_relaxed_stderr": 0.048241815132442176
},
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
"acc": 0.10282776349614396,
"acc_stderr": 0.010896388982494143,
"em_relaxed": 0.12853470437017994,
"em_relaxed_stderr": 0.012006729887990094
},
"e_tests.prompt3_qa_sk:informatika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.006923076923076923,
"qf1_stderr": 0.006424094288133824,
"rouge1": 0.00046479944765467505,
"rouge1_stderr": 0.0001876188412456943,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.00046479944765467505,
"rougeL_stderr": 0.0001876188412456943
},
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
"acc": 0.23026315789473684,
"acc_stderr": 0.03426059424403165,
"em_relaxed": 0.23026315789473684,
"em_relaxed_stderr": 0.03426059424403165
},
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
"acc": 0.27,
"acc_stderr": 0.04461960433384741,
"em_relaxed": 0.28,
"em_relaxed_stderr": 0.04512608598542127
},
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
"acc": 0.2145214521452145,
"acc_stderr": 0.023621042274383643,
"em_relaxed": 0.24092409240924093,
"em_relaxed_stderr": 0.0246081660183802
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
"acc": 0.2676767676767677,
"acc_stderr": 0.03154449888270286,
"em_relaxed": 0.2676767676767677,
"em_relaxed_stderr": 0.03154449888270286
},
"financial_exams.prompt3_completion_sk|0": {
"acc": 0.13343328335832083,
"acc_stderr": 0.009313609263721821,
"em_relaxed": 0.13343328335832083,
"em_relaxed_stderr": 0.009313609263721821
},
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
"acc": 0.19289340101522842,
"acc_stderr": 0.028183576131702342,
"em_relaxed": 0.19289340101522842,
"em_relaxed_stderr": 0.028183576131702342
},
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
"acc": 0.26900584795321636,
"acc_stderr": 0.0340105262010409,
"em_relaxed": 0.2807017543859649,
"em_relaxed_stderr": 0.034462962170884265
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
"acc": 0.15555555555555556,
"acc_stderr": 0.03130948364878314,
"em_relaxed": 0.15555555555555556,
"em_relaxed_stderr": 0.03130948364878314
},
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
"acc": 0.18,
"acc_stderr": 0.03861229196653695,
"em_relaxed": 0.18,
"em_relaxed_stderr": 0.03861229196653695
},
"scio.prompt3_mqa_single_correct_sk:aj|0": {
"acc": 0.5,
"acc_stderr": 0.16666666666666666,
"em_relaxed": 0.5,
"em_relaxed_stderr": 0.16666666666666666
},
"e_tests.prompt3_mqa_single_correct_sk:geografia|0": {
"acc": 0.22448979591836735,
"acc_stderr": 0.06022425581505364,
"em_relaxed": 0.22448979591836735,
"em_relaxed_stderr": 0.06022425581505364
},
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
"acc": 0.18181818181818182,
"acc_stderr": 0.03896103896103899,
"em_relaxed": 0.18181818181818182,
"em_relaxed_stderr": 0.03896103896103899
},
"e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
"acc": 0.15228426395939088,
"acc_stderr": 0.02566401503870399,
"em_relaxed": 0.15228426395939088,
"em_relaxed_stderr": 0.02566401503870399
},
"e_tests.prompt1_qa_sk:obcianska-nauka|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"e_tests.prompt3_mqa_single_correct_sk:matematika|0": {
"acc": 0.05128205128205128,
"acc_stderr": 0.035781575124326184,
"em_relaxed": 0.23076923076923078,
"em_relaxed_stderr": 0.0683479474337414
},
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
"acc": 0.26515151515151514,
"acc_stderr": 0.0272187441820065,
"em_relaxed": 0.2765151515151515,
"em_relaxed_stderr": 0.027580130232496353
},
"e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
"acc": 0.2,
"acc_stderr": 0.04020151261036846,
"em_relaxed": 0.22,
"em_relaxed_stderr": 0.04163331998932268
},
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
"acc": 0.19,
"acc_stderr": 0.03942772444036622,
"em_relaxed": 0.19,
"em_relaxed_stderr": 0.03942772444036622
},
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
"acc": 0.2206896551724138,
"acc_stderr": 0.03455930201924812,
"em_relaxed": 0.2413793103448276,
"em_relaxed_stderr": 0.03565998174135302
},
"sklep.prompt1_nli_3_classes_sk:nli|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
"acc": 0.18468468468468469,
"acc_stderr": 0.026102500004017645,
"em_relaxed": 0.1981981981981982,
"em_relaxed_stderr": 0.026815582053277203
},
"sklep.prompt3_nli_binary_sk:rte|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
"acc": 0.18624161073825504,
"acc_stderr": 0.015959806298600995,
"em_relaxed": 0.1912751677852349,
"em_relaxed_stderr": 0.01612394106868719
},
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
"acc": 0.22772277227722773,
"acc_stderr": 0.024131614233378947,
"em_relaxed": 0.264026402640264,
"em_relaxed_stderr": 0.0253659612714007
},
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
"acc": 0.19,
"acc_stderr": 0.039427724440366234,
"em_relaxed": 0.19,
"em_relaxed_stderr": 0.039427724440366234
},
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
"acc": 0.21,
"acc_stderr": 0.04093601807403326,
"em_relaxed": 0.22,
"em_relaxed_stderr": 0.04163331998932268
},
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
"acc": 0.24646983311938384,
"acc_stderr": 0.015450495390660532,
"em_relaxed": 0.24775353016688062,
"em_relaxed_stderr": 0.015477478391132707
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
"acc": 0.31666666666666665,
"acc_stderr": 0.034768900963930385,
"em_relaxed": 0.31666666666666665,
"em_relaxed_stderr": 0.034768900963930385
},
"arc.prompt2_mqa_single_correct_sk|0": {
"acc": 0.23353293413173654,
"acc_stderr": 0.012379394294094546,
"em_relaxed": 0.23353293413173654,
"em_relaxed_stderr": 0.012379394294094546
},
"slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": {
"acc": 0.14746543778801843,
"acc_stderr": 0.024125384514879104,
"em_relaxed": 0.14746543778801843,
"em_relaxed_stderr": 0.024125384514879104
},
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
"acc": 0.22,
"acc_stderr": 0.04163331998932269,
"em_relaxed": 0.22,
"em_relaxed_stderr": 0.04163331998932269
},
"slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": {
"acc": 0.11428571428571428,
"acc_stderr": 0.031197955091381097,
"em_relaxed": 0.11428571428571428,
"em_relaxed_stderr": 0.031197955091381097
},
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
"acc": 0.2359249329758713,
"acc_stderr": 0.02201321444385349,
"em_relaxed": 0.2359249329758713,
"em_relaxed_stderr": 0.02201321444385349
},
"e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": {
"acc": 0.017857142857142856,
"acc_stderr": 0.017857142857142867
},
"toxic_sk.prompt2_toxicity_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"e_tests.prompt2_mqa_single_correct_sk:fyzika|0": {
"acc": 0.13636363636363635,
"acc_stderr": 0.07488677009526491,
"em_relaxed": 0.22727272727272727,
"em_relaxed_stderr": 0.09144861547306321
},
"senti_sk.prompt1_sentiment_binary_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
"acc": 0.25288831835686776,
"acc_stderr": 0.015583584105316876,
"em_relaxed": 0.25417201540436457,
"em_relaxed_stderr": 0.01560965861721762
},
"slovak_tests.prompt2_qa_sk:monitor9|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
"acc": 0.23049645390070922,
"acc_stderr": 0.025123739226872395,
"em_relaxed": 0.23404255319148937,
"em_relaxed_stderr": 0.025257861359432386
},
"e_tests.prompt1_mqa_single_correct_sk:informatika|0": {
"acc": 0.264,
"acc_stderr": 0.03958494337416042,
"em_relaxed": 0.272,
"em_relaxed_stderr": 0.0399612715739433
},
"e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": {
"acc": 0.02142857142857143,
"acc_stderr": 0.012282463299882668
},
"e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
"acc": 0.2349799732977303,
"acc_stderr": 0.015502474593163047,
"em_relaxed": 0.2349799732977303,
"em_relaxed_stderr": 0.015502474593163047
},
"e_tests.prompt1_qa_sk:chemia|0": {
"qem": 0.018518518518518517,
"qem_stderr": 0.018518518518518517,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.00026455026455026457,
"rouge1_stderr": 0.00026455026455026484,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.00026455026455026457,
"rougeL_stderr": 0.00026455026455026484
},
"scio.prompt2_mqa_single_correct_sk:aj|0": {
"acc": 0.6,
"acc_stderr": 0.16329931618554522,
"em_relaxed": 0.6,
"em_relaxed_stderr": 0.16329931618554522
},
"e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
"acc": 0.19638242894056848,
"acc_stderr": 0.020220056896337255,
"em_relaxed": 0.20155038759689922,
"em_relaxed_stderr": 0.020418410166350903
},
"slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": {
"acc": 0.15176151761517614,
"acc_stderr": 0.018703199518943774,
"em_relaxed": 0.15447154471544716,
"em_relaxed_stderr": 0.01883928643967625
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
"acc": 0.37373737373737376,
"acc_stderr": 0.04887069039502487,
"em_relaxed": 0.3838383838383838,
"em_relaxed_stderr": 0.04912566964083466
},
"e_tests.prompt2_qa_sk:biologia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
"acc": 0.18,
"acc_stderr": 0.03861229196653697,
"em_relaxed": 0.18,
"em_relaxed_stderr": 0.03861229196653697
},
"e_tests.prompt1_qa_sk:informatika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.00824175824175824,
"qf1_stderr": 0.006643857376866051,
"rouge1": 0.00033664858861620243,
"rouge1_stderr": 0.00019799367371799002,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.00033664858861620243,
"rougeL_stderr": 0.00019799367371799002
},
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
"acc": 0.22784810126582278,
"acc_stderr": 0.03347528264060925,
"em_relaxed": 0.23417721518987342,
"em_relaxed_stderr": 0.03379766010403987
},
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"e_tests.prompt2_mqa_single_correct_sk:informatika|0": {
"acc": 0.224,
"acc_stderr": 0.037440705810827146,
"em_relaxed": 0.232,
"em_relaxed_stderr": 0.03790650637827826
},
"e_tests.prompt2_qa_sk:obcianska-nauka|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
"acc": 0.21,
"acc_stderr": 0.040936018074033256,
"em_relaxed": 0.21,
"em_relaxed_stderr": 0.040936018074033256
},
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
"acc": 0.23728813559322035,
"acc_stderr": 0.03933012549934385,
"em_relaxed": 0.23728813559322035,
"em_relaxed_stderr": 0.03933012549934385
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
"acc": 0.2488479262672811,
"acc_stderr": 0.029417385564063876,
"em_relaxed": 0.2626728110599078,
"em_relaxed_stderr": 0.029944068647646538
},
"e_tests.prompt3_qa_sk:fyzika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0014626760356334343,
"rouge1_stderr": 0.0010249453223271077,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0014626760356334343,
"rougeL_stderr": 0.0010249453223271077
},
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
"acc": 0.13247863247863248,
"acc_stderr": 0.022209309073165606,
"em_relaxed": 0.13247863247863248,
"em_relaxed_stderr": 0.022209309073165606
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
"acc": 0.19172932330827067,
"acc_stderr": 0.024182419147367247,
"em_relaxed": 0.19924812030075187,
"em_relaxed_stderr": 0.02453709604824174
},
"slovak_tests.prompt1_qa_sk:maturita|0": {
"qem": 0.01327433628318584,
"qem_stderr": 0.007629803632081149,
"qf1": 0.00454104547024901,
"qf1_stderr": 0.0026612932202592188,
"rouge1": 0.001362195307144739,
"rouge1_stderr": 0.0004933551702641079,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0012709627553702657,
"rougeL_stderr": 0.0004501834684090662
},
"e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"senti_sk.prompt3_sentiment_3_class_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
"acc": 0.1919191919191919,
"acc_stderr": 0.028057791672989017,
"em_relaxed": 0.1919191919191919,
"em_relaxed_stderr": 0.028057791672989017
},
"e_tests.prompt1_qa_sk:dejepis|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": {
"acc": 0.017857142857142856,
"acc_stderr": 0.017857142857142867
},
"senti_sk.prompt1_sentiment_3_class_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
"acc": 0.2323943661971831,
"acc_stderr": 0.03556905428023101,
"em_relaxed": 0.2323943661971831,
"em_relaxed_stderr": 0.03556905428023101
},
"slohry.prompt2_identity_form_sk:identity_form_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_declension_sk:declension_mid|0": {
"acc": 0.011976047904191617,
"acc_stderr": 0.005960989845848317,
"em_norm_relaxed": 0.011976047904191617,
"em_norm_relaxed_stderr": 0.005960989845848317
},
"slohry.prompt1_declension_sk:declension_low|0": {
"acc": 0.0029940119760479044,
"acc_stderr": 0.0029940119760478844,
"em_norm_relaxed": 0.0029940119760479044,
"em_norm_relaxed_stderr": 0.0029940119760478844
},
"analogies_sk.prompt1_analogy_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"slovak_trivia.prompt3_mqa_single_correct_sk|0": {
"acc": 0.22096317280453256,
"acc_stderr": 0.022114015575065166,
"em_relaxed": 0.2237960339943343,
"em_relaxed_stderr": 0.02221481955471835
},
"slohry.prompt3_identity_form_sk:identity_form_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_gradation_sk:gradation_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_inflection_sk:inflection_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_gradation_sk:gradation_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"chiby.prompt2_chiby_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_inflection_sk:inflection_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slovak_trivia.prompt2_mqa_single_correct_sk|0": {
"acc": 0.1501416430594901,
"acc_stderr": 0.01903936786327736,
"em_relaxed": 0.1558073654390935,
"em_relaxed_stderr": 0.019330515671326005
},
"chiby.prompt3_chiby_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_gradation_sk:gradation_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_declension_sk:declension_high|0": {
"acc": 0.017964071856287425,
"acc_stderr": 0.0072785348116838205,
"em_norm_relaxed": 0.017964071856287425,
"em_norm_relaxed_stderr": 0.0072785348116838205
},
"slohry.prompt2_inflection_sk:inflection_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_declension_sk:declension_low|0": {
"acc": 0.0029940119760479044,
"acc_stderr": 0.0029940119760478896,
"em_norm_relaxed": 0.0029940119760479044,
"em_norm_relaxed_stderr": 0.0029940119760478896
},
"slohry.prompt3_gradation_sk:gradation_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_gradation_sk:gradation_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"chiby.prompt1_chiby_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_declension_sk:declension_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_identity_form_sk:identity_form_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_identity_form_sk:identity_form_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_declension_sk:declension_high|0": {
"acc": 0.011976047904191617,
"acc_stderr": 0.005960989845848319,
"em_norm_relaxed": 0.011976047904191617,
"em_norm_relaxed_stderr": 0.005960989845848319
},
"slohry.prompt3_declension_sk:declension_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_inflection_sk:inflection_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slovak_trivia.prompt1_mqa_single_correct_sk|0": {
"acc": 0.22662889518413598,
"acc_stderr": 0.02231414649207437,
"em_relaxed": 0.23229461756373937,
"em_relaxed_stderr": 0.022508447165267422
},
"slohry.prompt1_identity_form_sk:identity_form_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_declension_sk:declension_high|0": {
"acc": 0.020958083832335328,
"acc_stderr": 0.007849722658097238,
"em_norm_relaxed": 0.020958083832335328,
"em_norm_relaxed_stderr": 0.007849722658097238
},
"slohry.prompt2_inflection_sk:inflection_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_declension_sk:declension_mid|0": {
"acc": 0.014970059880239521,
"acc_stderr": 0.006654483730236385,
"em_norm_relaxed": 0.014970059880239521,
"em_norm_relaxed_stderr": 0.006654483730236385
},
"analogies_sk.prompt2_analogy_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"slohry.prompt1_gradation_sk:gradation_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_gradation_sk:gradation_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_inflection_sk:inflection_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_identity_form_sk:identity_form_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_gradation_sk:gradation_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"analogies_sk.prompt3_analogy_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"slohry.prompt1_identity_form_sk:identity_form_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_identity_form_sk:identity_form_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_inflection_sk:inflection_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_inflection_sk:inflection_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_gradation_sk:gradation_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_inflection_sk:inflection_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_identity_form_sk:identity_form_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"e_tests_mqa_SK_AVERAGE|0": {
"em_relaxed": 0.21755858691850297,
"acc": 0.15008726003490402,
"em_relaxed_stderr": 0.03505820050399378,
"acc_stderr": 0.027287029108301525,
"em_relaxed__prompts": {
"1": 0.23819517313746066,
"2": 0.2088142707240294,
"3": 0.20566631689401887
},
"acc__prompts": {
"1": 0.16454749439042632,
"2": 0.14360508601346297,
"3": 0.14210919970082272
}
},
"e_tests_mqa_AVERAGE|0": {
"em_relaxed": 0.21755858691850297,
"em_relaxed_stderr": 0.03505820050399378,
"acc": 0.15008726003490402,
"acc_stderr": 0.027287029108301525
},
"mmlu_SK_AVERAGE|0": {
"em_relaxed": 0.22816822334507222,
"acc": 0.2224518960853366,
"em_relaxed_stderr": 0.025975113405158696,
"acc_stderr": 0.02571374453987217,
"em_relaxed__prompts": {
"1": 0.24628693921298422,
"2": 0.21719491655182974,
"3": 0.22102281427040268
},
"acc__prompts": {
"1": 0.2399326290001531,
"2": 0.21137651201959884,
"3": 0.21604654723625785
}
},
"mmlu_AVERAGE|0": {
"em_relaxed": 0.22816822334507222,
"em_relaxed_stderr": 0.025975113405158696,
"acc": 0.2224518960853366,
"acc_stderr": 0.02571374453987217
},
"scio_SK_AVERAGE|0": {
"em_relaxed": 0.24911297852474323,
"acc": 0.238468720821662,
"em_relaxed_stderr": 0.020694508860887174,
"acc_stderr": 0.0204222231602884,
"em_relaxed__prompts": {
"1": 0.261624649859944,
"3": 0.23865546218487396,
"2": 0.24705882352941178
},
"acc__prompts": {
"1": 0.2504201680672269,
"3": 0.2280112044817927,
"2": 0.23697478991596638
}
},
"scio_AVERAGE|0": {
"em_relaxed": 0.24911297852474323,
"em_relaxed_stderr": 0.020694508860887174,
"acc": 0.238468720821662,
"acc_stderr": 0.0204222231602884
},
"slovak_tests_qa_SK_AVERAGE|0": {
"rouge2": 0.0,
"rouge1": 0.0011809129165421361,
"rougeL": 0.0011384377140699951,
"qf1": 0.003009101538513303,
"qem": 0.012605042016806723,
"rouge2_stderr": 0.0,
"rouge1_stderr": 0.0007262749376038493,
"rougeL_stderr": 0.0006990122730613884,
"qf1_stderr": 0.0019761043274910683,
"qem_stderr": 0.0072451076506316785,
"rouge2__prompts": {
"3": 0.0,
"2": 0.0,
"1": 0.0
},
"rouge1__prompts": {
"3": 0.001309304483429979,
"2": 0.0008896015463691078,
"1": 0.001343832719827322
},
"rougeL__prompts": {
"3": 0.001309304483429979,
"2": 0.0008488085301334873,
"1": 0.0012572001286465196
},
"qf1__prompts": {
"3": 0.0021008403361344537,
"2": 0.0026143790849673205,
"1": 0.004312085194438136
},
"qem__prompts": {
"3": 0.012605042016806723,
"2": 0.012605042016806723,
"1": 0.012605042016806723
}
},
"slovak_tests_qa_AVERAGE|0": {
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rouge1": 0.0011809129165421361,
"rouge1_stderr": 0.0007262749376038493,
"rougeL": 0.0011384377140699951,
"rougeL_stderr": 0.0006990122730613884,
"qf1": 0.003009101538513303,
"qf1_stderr": 0.0019761043274910683,
"qem": 0.012605042016806723,
"qem_stderr": 0.0072451076506316785
},
"slovak_tests_mqa_SK_AVERAGE|0": {
"em_relaxed": 0.17543859649122806,
"acc": 0.17389966143428748,
"em_relaxed_stderr": 0.02232972578404266,
"acc_stderr": 0.02225738292492629,
"em_relaxed__prompts": {
"1": 0.20498614958448755,
"3": 0.17820867959372114,
"2": 0.14312096029547552
},
"acc__prompts": {
"1": 0.20221606648199447,
"3": 0.17820867959372114,
"2": 0.14127423822714683
}
},
"slovak_tests_mqa_AVERAGE|0": {
"em_relaxed": 0.17543859649122806,
"em_relaxed_stderr": 0.02232972578404266,
"acc": 0.17389966143428748,
"acc_stderr": 0.02225738292492629
},
"movie_review_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"2": 0.0,
"3": 0.0,
"1": 0.0
},
"acc__prompts": {
"2": 0.0,
"3": 0.0,
"1": 0.0
}
},
"movie_review_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"e_tests_qa_SK_AVERAGE|0": {
"rouge2": 0.0,
"rouge1": 0.00029664081444656757,
"rougeL": 0.00029664081444656757,
"qf1": 0.0026099107814683436,
"qem": 0.002257336343115124,
"rouge2_stderr": 0.0,
"rouge1_stderr": 0.0001935712320593315,
"rougeL_stderr": 0.0001935712320593315,
"qf1_stderr": 0.002326519969477271,
"qem_stderr": 0.002257336343115124,
"rouge2__prompts": {
"1": 0.0,
"2": 0.0,
"3": 0.0
},
"rouge1__prompts": {
"1": 0.00011697710542177868,
"2": 0.0002733713466381956,
"3": 0.0004995739912797284
},
"rougeL__prompts": {
"1": 0.00011697710542177868,
"2": 0.0002733713466381956,
"3": 0.0004995739912797284
},
"qf1__prompts": {
"1": 0.0025798129635601417,
"2": 0.00290228958400516,
"3": 0.002347629796839729
},
"qem__prompts": {
"1": 0.002257336343115124,
"2": 0.002257336343115124,
"3": 0.002257336343115124
}
},
"e_tests_qa_AVERAGE|0": {
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rouge1": 0.00029664081444656757,
"rouge1_stderr": 0.0001935712320593315,
"rougeL": 0.00029664081444656757,
"rougeL_stderr": 0.0001935712320593315,
"qf1": 0.0026099107814683436,
"qf1_stderr": 0.002326519969477271,
"qem": 0.002257336343115124,
"qem_stderr": 0.002257336343115124
},
"sklep_nli_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"3": 0.0,
"2": 0.0,
"1": 0.0
},
"acc__prompts": {
"3": 0.0,
"2": 0.0,
"1": 0.0
}
},
"sklep_nli_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"sklep_sentiment_analysis_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"2": 0.0,
"1": 0.0,
"3": 0.0
},
"acc__prompts": {
"2": 0.0,
"1": 0.0,
"3": 0.0
}
},
"sklep_sentiment_analysis_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"arc_SK_AVERAGE|0": {
"em_relaxed": 0.24493869404049046,
"acc": 0.24493869404049046,
"em_relaxed_stderr": 0.012565550837641016,
"acc_stderr": 0.012565550837641016,
"em_relaxed__prompts": {
"1": 0.2728828058169376,
"3": 0.22840034217279725,
"2": 0.23353293413173654
},
"acc__prompts": {
"1": 0.2728828058169376,
"3": 0.22840034217279725,
"2": 0.23353293413173654
}
},
"arc_AVERAGE|0": {
"em_relaxed": 0.24493869404049046,
"em_relaxed_stderr": 0.012565550837641016,
"acc": 0.24493869404049046,
"acc_stderr": 0.012565550837641016
},
"belebele_SK_AVERAGE|0": {
"em_relaxed": 0.2611111111111111,
"acc": 0.2537037037037037,
"em_relaxed_stderr": 0.014646428295956382,
"acc_stderr": 0.014509896842640889,
"em_relaxed__prompts": {
"2": 0.25666666666666665,
"3": 0.2544444444444444,
"1": 0.2722222222222222
},
"acc__prompts": {
"2": 0.2511111111111111,
"3": 0.24666666666666667,
"1": 0.2633333333333333
}
},
"belebele_AVERAGE|0": {
"em_relaxed": 0.2611111111111111,
"em_relaxed_stderr": 0.014646428295956382,
"acc": 0.2537037037037037,
"acc_stderr": 0.014509896842640889
},
"senti_sk_3class_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"2": 0.0,
"3": 0.0,
"1": 0.0
},
"acc__prompts": {
"2": 0.0,
"3": 0.0,
"1": 0.0
}
},
"senti_sk_3class_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"sklep_hate_speech_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"2": 0.0,
"1": 0.0,
"3": 0.0
},
"acc__prompts": {
"2": 0.0,
"1": 0.0,
"3": 0.0
}
},
"sklep_hate_speech_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"slovakcopa_SK_AVERAGE|0": {
"em_relaxed": 0.05333333333333334,
"acc": 0.05333333333333334,
"em_relaxed_stderr": 0.007876087666852084,
"acc_stderr": 0.007876087666852084,
"em_relaxed__prompts": {
"3": 0.144,
"2": 0.006,
"1": 0.01
},
"acc__prompts": {
"3": 0.144,
"2": 0.006,
"1": 0.01
}
},
"slovakcopa_AVERAGE|0": {
"em_relaxed": 0.05333333333333334,
"em_relaxed_stderr": 0.007876087666852084,
"acc": 0.05333333333333334,
"acc_stderr": 0.007876087666852084
},
"globalmmlu_lite_SK_AVERAGE|0": {
"em_relaxed": 0.21333333333333335,
"acc": 0.21083333333333334,
"em_relaxed_stderr": 0.020468272436489743,
"acc_stderr": 0.020379605193633,
"em_relaxed__prompts": {
"1": 0.2425,
"3": 0.205,
"2": 0.1925
},
"acc__prompts": {
"1": 0.24,
"3": 0.2025,
"2": 0.19
}
},
"globalmmlu_lite_AVERAGE|0": {
"em_relaxed": 0.21333333333333335,
"em_relaxed_stderr": 0.020468272436489743,
"acc": 0.21083333333333334,
"acc_stderr": 0.020379605193633
},
"sklep_qa_SK_AVERAGE|0": {
"qf1": 0.0070376260418807705,
"qem": 0.00027827054854081884,
"qf1_stderr": 0.0003824771262616922,
"qem_stderr": 0.0001679351079586058,
"qf1__prompts": {
"3": 0.006093522810123945,
"2": 0.006743145308527306,
"1": 0.008276210006991061
},
"qem__prompts": {
"3": 0.0002087029114056141,
"2": 0.0002087029114056141,
"1": 0.0004174058228112282
}
},
"sklep_qa_AVERAGE|0": {
"qf1": 0.0070376260418807705,
"qf1_stderr": 0.0003824771262616922,
"qem": 0.00027827054854081884,
"qem_stderr": 0.0001679351079586058
},
"financial_exams_SK_AVERAGE|0": {
"em_relaxed": 0.08995502248875563,
"acc": 0.0887056471764118,
"em_relaxed_stderr": 0.007689948855002123,
"acc_stderr": 0.007626104510560348,
"em_relaxed__prompts": {
"1": 0.05697151424287856,
"2": 0.07946026986506746,
"3": 0.13343328335832083
},
"acc__prompts": {
"1": 0.053973013493253376,
"2": 0.07871064467766117,
"3": 0.13343328335832083
}
},
"financial_exams_AVERAGE|0": {
"em_relaxed": 0.08995502248875563,
"em_relaxed_stderr": 0.007689948855002123,
"acc": 0.0887056471764118,
"acc_stderr": 0.007626104510560348
},
"ifeval_sk_SK_AVERAGE|0": {
"prompt_level_strict_acc": 0.0782122905027933,
"prompt_level_loose_acc": 0.0782122905027933,
"inst_level_strict_acc": 0.17312348668280872,
"inst_level_loose_acc": 0.17312348668280872,
"prompt_level_strict_acc_stderr": 0.011597665111537078,
"prompt_level_loose_acc_stderr": 0.011597665111537078,
"inst_level_strict_acc_stderr": 0.0004210039776560394,
"inst_level_loose_acc_stderr": 0.0004210039776560394
},
"ifeval_sk_AVERAGE|0": {
"prompt_level_strict_acc": 0.0782122905027933,
"prompt_level_strict_acc_stderr": 0.011597665111537078,
"prompt_level_loose_acc": 0.0782122905027933,
"prompt_level_loose_acc_stderr": 0.011597665111537078,
"inst_level_strict_acc": 0.17312348668280872,
"inst_level_strict_acc_stderr": 0.0004210039776560394,
"inst_level_loose_acc": 0.17312348668280872,
"inst_level_loose_acc_stderr": 0.0004210039776560394
},
"toxic_sk_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"1": 0.0,
"3": 0.0,
"2": 0.0
},
"acc__prompts": {
"1": 0.0,
"3": 0.0,
"2": 0.0
}
},
"toxic_sk_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"driving_test_SK_AVERAGE|0": {
"em_relaxed": 0.1407513076557299,
"acc": 0.13742272943414172,
"em_relaxed_stderr": 0.013052324431028241,
"acc_stderr": 0.012921192285229888,
"em_relaxed__prompts": {
"2": 0.10271041369472182,
"1": 0.16833095577746077,
"3": 0.15121255349500715
},
"acc__prompts": {
"2": 0.09985734664764621,
"1": 0.16405135520684735,
"3": 0.14835948644793154
}
},
"driving_test_AVERAGE|0": {
"em_relaxed": 0.1407513076557299,
"em_relaxed_stderr": 0.013052324431028241,
"acc": 0.13742272943414172,
"acc_stderr": 0.012921192285229888
},
"sklep_rte_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"1": 0.0,
"2": 0.0,
"3": 0.0
},
"acc__prompts": {
"1": 0.0,
"2": 0.0,
"3": 0.0
}
},
"sklep_rte_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"multi_wiki_qa_SK_AVERAGE|0": {
"qf1": 0.005263913929238519,
"qem": 0.0005336535254486025,
"qf1_stderr": 0.0005228307644937593,
"qem_stderr": 0.00032536056243172176,
"qf1__prompts": {
"3": 0.005765729879562343,
"2": 0.005417769336679904,
"1": 0.004608242571473309
},
"qem__prompts": {
"3": 0.0006003602161296778,
"2": 0.0006003602161296778,
"1": 0.00040024014408645187
}
},
"multi_wiki_qa_AVERAGE|0": {
"qf1": 0.005263913929238519,
"qf1_stderr": 0.0005228307644937593,
"qem": 0.0005336535254486025,
"qem_stderr": 0.00032536056243172176
},
"truthful_qa_SK_AVERAGE|0": {
"em_relaxed": 0.09682947729220223,
"acc": 0.06469580119965725,
"em_relaxed_stderr": 0.010525690171591523,
"acc_stderr": 0.008619652748939895,
"em_relaxed__prompts": {
"2": 0.08354755784061697,
"1": 0.07840616966580977,
"3": 0.12853470437017994
},
"acc__prompts": {
"2": 0.04884318766066838,
"1": 0.042416452442159386,
"3": 0.10282776349614396
}
},
"truthful_qa_AVERAGE|0": {
"em_relaxed": 0.09682947729220223,
"em_relaxed_stderr": 0.010525690171591523,
"acc": 0.06469580119965725,
"acc_stderr": 0.008619652748939895
},
"senti_sk_binary_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"3": 0.0,
"2": 0.0,
"1": 0.0
},
"acc__prompts": {
"3": 0.0,
"2": 0.0,
"1": 0.0
}
},
"senti_sk_binary_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"skj1_SK_AVERAGE|0": {
"em_relaxed": 0.61,
"acc": 0.5966666666666667,
"em_relaxed_stderr": 0.04721666475945084,
"acc_stderr": 0.04715805991356215,
"em_relaxed__prompts": {
"1": 0.44,
"3": 0.75,
"2": 0.64
},
"acc__prompts": {
"1": 0.41,
"3": 0.75,
"2": 0.63
}
},
"skj1_AVERAGE|0": {
"em_relaxed": 0.61,
"em_relaxed_stderr": 0.04721666475945084,
"acc": 0.5966666666666667,
"acc_stderr": 0.04715805991356215
},
"slohry_SK_AVERAGE|0": {
"em_norm_relaxed": 0.002331002331002331,
"acc": 0.002331002331002331,
"em_norm_relaxed_stderr": 0.0011036777204322752,
"acc_stderr": 0.0011036777204322752,
"em_norm_relaxed__prompts": {
"2": 0.003246753246753247,
"1": 0.0027472527472527475,
"3": 0.000999000999000999
},
"acc__prompts": {
"2": 0.003246753246753247,
"1": 0.0027472527472527475,
"3": 0.000999000999000999
}
},
"slohry_AVERAGE|0": {
"em_norm_relaxed": 0.002331002331002331,
"em_norm_relaxed_stderr": 0.0011036777204322752,
"acc": 0.002331002331002331,
"acc_stderr": 0.0011036777204322752
},
"analogies_sk_SK_AVERAGE|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"acc__prompts": {
"1": 0.0,
"2": 0.0,
"3": 0.0
}
},
"analogies_sk_AVERAGE|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"slovak_trivia_SK_AVERAGE|0": {
"em_relaxed": 0.20396600566572237,
"acc": 0.19924457034938622,
"em_relaxed_stderr": 0.021351260797103925,
"acc_stderr": 0.021155843310138964,
"em_relaxed__prompts": {
"3": 0.2237960339943343,
"2": 0.1558073654390935,
"1": 0.23229461756373937
},
"acc__prompts": {
"3": 0.22096317280453256,
"2": 0.1501416430594901,
"1": 0.22662889518413598
}
},
"slovak_trivia_AVERAGE|0": {
"em_relaxed": 0.20396600566572237,
"em_relaxed_stderr": 0.021351260797103925,
"acc": 0.19924457034938622,
"acc_stderr": 0.021155843310138964
},
"chiby_SK_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"acc": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed__prompts": {
"2": 0.0,
"3": 0.0,
"1": 0.0
},
"acc__prompts": {
"2": 0.0,
"3": 0.0,
"1": 0.0
}
},
"chiby_AVERAGE|0": {
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0,
"acc": 0.0,
"acc_stderr": 0.0
},
"SK_AVERAGE|0": {
"em_relaxed": 0.21419205155386342,
"acc": 0.10986595916434654,
"rouge2": 0.0,
"rouge1": 0.0007387768654943519,
"rougeL": 0.0007175392642582814,
"qf1": 0.004480138072775234,
"qem": 0.003918575608477817,
"em_norm_relaxed": 0.0002331002331002331,
"prompt_level_strict_acc": 0.0782122905027933,
"prompt_level_loose_acc": 0.0782122905027933,
"inst_level_strict_acc": 0.17312348668280872,
"inst_level_loose_acc": 0.17312348668280872,
"em_relaxed_stderr": 0.006300951163940428,
"acc_stderr": 0.003263889036835735,
"rouge2_stderr": 0.0,
"rouge1_stderr": 0.00037581415183320424,
"rougeL_stderr": 0.0003626596130572774,
"qf1_stderr": 0.0007801171483309406,
"qem_stderr": 0.0018993619641089511,
"em_norm_relaxed_stderr": 0.00011036777204322752,
"prompt_level_strict_acc_stderr": 0.011597665111537078,
"prompt_level_loose_acc_stderr": 0.011597665111537078,
"inst_level_strict_acc_stderr": 0.0004210039776560394,
"inst_level_loose_acc_stderr": 0.0004210039776560394
},
"AVERAGE|0": {
"em_relaxed": 0.18120805585001873,
"acc": 0.09273256151262048,
"rouge2": 0.0,
"rouge1": 0.0007387768654943519,
"rougeL": 0.0007175392642582814,
"qf1": 0.004480138072775234,
"qem": 0.003918575608477817,
"em_norm_relaxed": 0.000259000259000259,
"prompt_level_strict_acc": 0.0782122905027933,
"prompt_level_loose_acc": 0.0782122905027933,
"inst_level_strict_acc": 0.17312348668280872,
"inst_level_loose_acc": 0.17312348668280872,
"em_relaxed_stderr": 0.005577871787987613,
"acc_stderr": 0.002843082460505095,
"rouge2_stderr": 0.0,
"rouge1_stderr": 0.00037581415183320424,
"rougeL_stderr": 0.0003626596130572774,
"qf1_stderr": 0.0007801171483309406,
"qem_stderr": 0.0018993619641089511,
"em_norm_relaxed_stderr": 0.00012263085782580836,
"prompt_level_strict_acc_stderr": 0.011597665111537078,
"prompt_level_loose_acc_stderr": 0.011597665111537078,
"inst_level_strict_acc_stderr": 0.0004210039776560394,
"inst_level_loose_acc_stderr": 0.0004210039776560394
}
}
}