Diplomovka/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-last-109064.json
2026-11-12 16:51:42 +01:00

3250 lines
112 KiB
JSON

{
"model_name": "models/ministral-last-109064",
"config_general": {
"lighteval_sha": "?",
"num_fewshot_seeds": 1,
"max_samples": null,
"job_id": "0",
"start_time": 883355.295419078,
"end_time": 883503.394399547,
"total_evaluation_time_secondes": "148.0989804690471",
"model_config": {
"model_name": "models/ministral-last-109064",
"generation_parameters": {
"num_blocks": null,
"block_size": null,
"early_stopping": null,
"repetition_penalty": null,
"frequency_penalty": null,
"length_penalty": null,
"presence_penalty": null,
"max_new_tokens": null,
"min_new_tokens": null,
"seed": null,
"stop_tokens": null,
"temperature": 0,
"top_k": null,
"min_p": null,
"top_p": null,
"truncate_prompt": null,
"cache_implementation": null,
"response_format": null,
"skip_special_tokens": true
},
"system_prompt": null,
"cache_dir": "~/.cache/huggingface/lighteval",
"tokenizer": null,
"revision": "main",
"dtype": "bfloat16",
"tensor_parallel_size": 1,
"data_parallel_size": 1,
"pipeline_parallel_size": 1,
"gpu_memory_utilization": 0.9,
"enable_prefix_caching": null,
"max_model_length": null,
"quantization": null,
"load_format": null,
"swap_space": 4,
"seed": 1234,
"trust_remote_code": true,
"add_special_tokens": true,
"multichoice_continuations_start_space": true,
"pairwise_tokenization": false,
"max_num_seqs": 128,
"max_num_batched_tokens": 2048,
"subfolder": null,
"is_async": false,
"override_chat_template": null,
"enforce_eager": false
},
"model_name": "models/ministral-last-109064"
},
"versions": {},
"results": {
"movie_review.prompt1_sentiment_binary_sk|0": {
"acc": 0.09860557768924302,
"acc_stderr": 0.006654792030522998,
"em_norm_relaxed": 0.10109561752988047,
"em_norm_relaxed_stderr": 0.006728979843112432
},
"e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": {
"acc": 0.1,
"acc_stderr": 0.06882472016116853
},
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
"acc": 0.21983914209115282,
"acc_stderr": 0.021472030675572816,
"em_relaxed": 0.21983914209115282,
"em_relaxed_stderr": 0.021472030675572816
},
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
"acc": 0.27350427350427353,
"acc_stderr": 0.029202540153431183,
"em_relaxed": 0.27350427350427353,
"em_relaxed_stderr": 0.029202540153431183
},
"belebele.prompt3_mqa_single_correct_sk|0": {
"acc": 0.27111111111111114,
"acc_stderr": 0.014826016446581967,
"em_relaxed": 0.2722222222222222,
"em_relaxed_stderr": 0.014845038794433947
},
"e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": {
"acc": 0.2318840579710145,
"acc_stderr": 0.05117930441535768,
"em_relaxed": 0.2318840579710145,
"em_relaxed_stderr": 0.05117930441535768
},
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
"acc": 0.254601226993865,
"acc_stderr": 0.024164783161712056,
"em_relaxed": 0.254601226993865,
"em_relaxed_stderr": 0.024164783161712056
},
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
"acc": 0.20202020202020202,
"acc_stderr": 0.0405583530232473,
"em_relaxed": 0.2222222222222222,
"em_relaxed_stderr": 0.041996052556580815
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
"acc": 0.20522388059701493,
"acc_stderr": 0.017460611985170203,
"em_relaxed": 0.20522388059701493,
"em_relaxed_stderr": 0.017460611985170203
},
"senti_sk.prompt2_sentiment_binary_sk|0": {
"acc": 0.32135193133047213,
"acc_stderr": 0.007649500362664599,
"em_norm_relaxed": 0.3261802575107296,
"em_norm_relaxed_stderr": 0.007679288875174346
},
"slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": {
"acc": 0.18095238095238095,
"acc_stderr": 0.03775026958386237,
"em_relaxed": 0.18095238095238095,
"em_relaxed_stderr": 0.03775026958386237
},
"e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": {
"acc": 0.05357142857142857,
"acc_stderr": 0.030361917118846844
},
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
"acc": 0.27181208053691275,
"acc_stderr": 0.018238857466939602,
"em_relaxed": 0.27181208053691275,
"em_relaxed_stderr": 0.018238857466939602
},
"e_tests.prompt3_qa_sk:chemia|0": {
"qem": 0.037037037037037035,
"qem_stderr": 0.025940896147128693,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.02444444444444445,
"rouge1_stderr": 0.011223747742795042,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.02444444444444445,
"rougeL_stderr": 0.011223747742795042
},
"scio.prompt3_mqa_single_correct_sk:vsp|0": {
"acc": 0.2657766990291262,
"acc_stderr": 0.015398294286338463,
"em_relaxed": 0.27184466019417475,
"em_relaxed_stderr": 0.015508596797488939
},
"arc.prompt2_mqa_single_correct_sk|0": {
"acc": 0.24123182207014543,
"acc_stderr": 0.01251844632394384,
"em_relaxed": 0.24721984602224123,
"em_relaxed_stderr": 0.012622759999132595
},
"sklep.prompt3_nli_binary_sk:rte|0": {
"acc": 0.1650602409638554,
"acc_stderr": 0.009114348186638069,
"em_norm_relaxed": 0.16566265060240964,
"em_norm_relaxed_stderr": 0.009127670467992892
},
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
"acc": 0.1513157894736842,
"acc_stderr": 0.029162631596843982,
"em_relaxed": 0.1513157894736842,
"em_relaxed_stderr": 0.029162631596843982
},
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
"acc": 0.25892857142857145,
"acc_stderr": 0.041577515398656284,
"em_relaxed": 0.2767857142857143,
"em_relaxed_stderr": 0.04246624336697624
},
"driving_test.prompt3_mqa_single_correct_sk|0": {
"acc": 0.17261055634807418,
"acc_stderr": 0.014283664605079543,
"em_relaxed": 0.17261055634807418,
"em_relaxed_stderr": 0.014283664605079543
},
"e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
"acc": 0.23232323232323232,
"acc_stderr": 0.04266016017054689,
"em_relaxed": 0.23232323232323232,
"em_relaxed_stderr": 0.04266016017054689
},
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
"acc": 0.24679760888129804,
"acc_stderr": 0.01260472537669998,
"em_relaxed": 0.2493595217762596,
"em_relaxed_stderr": 0.012648412983648835
},
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
"acc": 0.25217391304347825,
"acc_stderr": 0.040672227541547167,
"em_relaxed": 0.2608695652173913,
"em_relaxed_stderr": 0.04112631751856164
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
"acc": 0.2676767676767677,
"acc_stderr": 0.03154449888270286,
"em_relaxed": 0.2676767676767677,
"em_relaxed_stderr": 0.03154449888270286
},
"senti_sk.prompt1_sentiment_binary_sk|0": {
"acc": 0.39914163090128757,
"acc_stderr": 0.0080217638856134,
"em_norm_relaxed": 0.4069206008583691,
"em_norm_relaxed_stderr": 0.008046954795106477
},
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
"acc": 0.1833013435700576,
"acc_stderr": 0.011991910538467281,
"em_norm_relaxed": 0.19193857965451055,
"em_norm_relaxed_stderr": 0.012206129123492155
},
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
"acc": 0.2222222222222222,
"acc_stderr": 0.03188578017686398,
"em_relaxed": 0.2222222222222222,
"em_relaxed_stderr": 0.03188578017686398
},
"e_tests.prompt2_qa_sk:fyzika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.014285714285714287,
"rouge1_stderr": 0.014285714285714287,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.014285714285714287,
"rougeL_stderr": 0.014285714285714287
},
"scio.prompt2_mqa_single_correct_sk:che|0": {
"acc": 0.2601626016260163,
"acc_stderr": 0.028028995361669362,
"em_relaxed": 0.2601626016260163,
"em_relaxed_stderr": 0.028028995361669362
},
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
"acc": 0.17,
"acc_stderr": 0.0377525168068637,
"em_relaxed": 0.17,
"em_relaxed_stderr": 0.0377525168068637
},
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
"acc": 0.2408198121263877,
"acc_stderr": 0.012500448888138267,
"em_relaxed": 0.2408198121263877,
"em_relaxed_stderr": 0.012500448888138267
},
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
"acc": 0.25153374233128833,
"acc_stderr": 0.024068141649530538,
"em_relaxed": 0.25153374233128833,
"em_relaxed_stderr": 0.024068141649530538
},
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
"acc": 0.22826086956521738,
"acc_stderr": 0.025309563050037266,
"em_relaxed": 0.22826086956521738,
"em_relaxed_stderr": 0.025309563050037266
},
"e_tests.prompt2_qa_sk:geografia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.01904761904761905,
"qf1_stderr": 0.019047619047619056,
"rouge1": 0.01360544217687075,
"rouge1_stderr": 0.013605442176870748,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.01360544217687075,
"rougeL_stderr": 0.013605442176870748
},
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
"acc": 0.29292929292929293,
"acc_stderr": 0.045972676254181784,
"em_relaxed": 0.29292929292929293,
"em_relaxed_stderr": 0.045972676254181784
},
"e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": {
"acc": 0.05,
"acc_stderr": 0.04999999999999998
},
"e_tests.prompt2_mqa_single_correct_sk:fyzika|0": {
"acc": 0.13636363636363635,
"acc_stderr": 0.07488677009526491,
"em_relaxed": 0.13636363636363635,
"em_relaxed_stderr": 0.07488677009526491
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
"acc": 0.24067164179104478,
"acc_stderr": 0.018482060138670608,
"em_relaxed": 0.24253731343283583,
"em_relaxed_stderr": 0.01853075065058218
},
"slovakcopa.prompt1_copa_sk|0": {
"acc": 0.402,
"acc_stderr": 0.021948929609938612,
"em_relaxed": 0.402,
"em_relaxed_stderr": 0.021948929609938612
},
"movie_review.prompt3_sentiment_binary_sk|0": {
"acc": 0.20766932270916336,
"acc_stderr": 0.009054526051113992,
"em_norm_relaxed": 0.20916334661354583,
"em_norm_relaxed_stderr": 0.009078466587423147
},
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
"acc": 0.20348837209302326,
"acc_stderr": 0.03078703062133896,
"em_relaxed": 0.20348837209302326,
"em_relaxed_stderr": 0.03078703062133896
},
"e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": {
"acc": 0.034482758620689655,
"acc_stderr": 0.034482758620689676
},
"e_tests.prompt3_mqa_single_correct_sk:geografia|0": {
"acc": 0.20408163265306123,
"acc_stderr": 0.05817221556628253,
"em_relaxed": 0.20408163265306123,
"em_relaxed_stderr": 0.05817221556628253
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
"acc": 0.22932330827067668,
"acc_stderr": 0.02582481829225878,
"em_relaxed": 0.22932330827067668,
"em_relaxed_stderr": 0.02582481829225878
},
"sklep.prompt2_qa_sk:question-answering|0": {
"qem": 0.05238443076280914,
"qem_stderr": 0.0022760883977866997,
"qf1": 0.14428713100967,
"qf1_stderr": 0.002729924096699559
},
"e_tests.prompt2_qa_sk:informatika|0": {
"qem": 0.038461538461538464,
"qem_stderr": 0.021915495473007307,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0036324786324786326,
"rouge1_stderr": 0.0025563439430967027,
"rouge2": 0.0008547008547008547,
"rouge2_stderr": 0.0008547008547008546,
"rougeL": 0.0036324786324786326,
"rougeL_stderr": 0.0025563439430967027
},
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
"acc": 0.32340425531914896,
"acc_stderr": 0.030579442773610337,
"em_relaxed": 0.32340425531914896,
"em_relaxed_stderr": 0.030579442773610337
},
"e_tests.prompt1_qa_sk:dejepis|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
"acc": 0.2482758620689655,
"acc_stderr": 0.03600105692727771,
"em_relaxed": 0.2482758620689655,
"em_relaxed_stderr": 0.03600105692727771
},
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
"acc": 0.3926940639269406,
"acc_stderr": 0.03307521564995785,
"em_relaxed": 0.3926940639269406,
"em_relaxed_stderr": 0.03307521564995785
},
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
"acc": 0.31,
"acc_stderr": 0.04648231987117316,
"em_relaxed": 0.31,
"em_relaxed_stderr": 0.04648231987117316
},
"e_tests.prompt1_qa_sk:obcianska-nauka|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.020408163265306124,
"rouge1_stderr": 0.020408163265306124,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.020408163265306124,
"rougeL_stderr": 0.020408163265306124
},
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
"acc": 0.2548262548262548,
"acc_stderr": 0.027129450992473495,
"em_relaxed": 0.2625482625482625,
"em_relaxed_stderr": 0.027394382974098772
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
"acc": 0.19626168224299065,
"acc_stderr": 0.027213555922450262,
"em_relaxed": 0.205607476635514,
"em_relaxed_stderr": 0.027691547344010747
},
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
"acc": 0.23478260869565218,
"acc_stderr": 0.039698395317531235,
"em_relaxed": 0.23478260869565218,
"em_relaxed_stderr": 0.039698395317531235
},
"senti_sk.prompt3_sentiment_3_class_sk|0": {
"acc": 0.573444515715202,
"acc_stderr": 0.007232629453324713,
"em_norm_relaxed": 0.5809279452640581,
"em_norm_relaxed_stderr": 0.007215529950821413
},
"e_tests.prompt1_qa_sk:biologia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.019047619047619046,
"qf1_stderr": 0.019047619047619035,
"rouge1": 0.04,
"rouge1_stderr": 0.022529159443043678,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.04,
"rougeL_stderr": 0.022529159443043678
},
"slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": {
"acc": 0.27371273712737126,
"acc_stderr": 0.023242230741504617,
"em_relaxed": 0.27371273712737126,
"em_relaxed_stderr": 0.023242230741504617
},
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
"acc": 0.35365853658536583,
"acc_stderr": 0.03744805613781578,
"em_relaxed": 0.35365853658536583,
"em_relaxed_stderr": 0.03744805613781578
},
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
"acc": 0.23857868020304568,
"acc_stderr": 0.03044390069569734,
"em_relaxed": 0.23857868020304568,
"em_relaxed_stderr": 0.03044390069569734
},
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
"acc": 0.3926940639269406,
"acc_stderr": 0.03307521564995785,
"em_relaxed": 0.3926940639269406,
"em_relaxed_stderr": 0.03307521564995785
},
"e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": {
"acc": 0.022727272727272728,
"acc_stderr": 0.022727272727272728
},
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
"acc": 0.2926829268292683,
"acc_stderr": 0.035637888362588285,
"em_relaxed": 0.2926829268292683,
"em_relaxed_stderr": 0.035637888362588285
},
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
"acc": 0.2323943661971831,
"acc_stderr": 0.03556905428023102,
"em_relaxed": 0.23943661971830985,
"em_relaxed_stderr": 0.03593796120702438
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
"acc": 0.2534562211981567,
"acc_stderr": 0.02959731008909016,
"em_relaxed": 0.2534562211981567,
"em_relaxed_stderr": 0.02959731008909016
},
"multi_wiki_qa.prompt3_qa_sk|0": {
"qem": 0.07884730838503101,
"qem_stderr": 0.0038128351969214953,
"qf1": 0.12171966052606027,
"qf1_stderr": 0.003955037338259647
},
"e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": {
"acc": 0.02857142857142857,
"acc_stderr": 0.014130710990583526
},
"e_tests.prompt2_mqa_single_correct_sk:biologia|0": {
"acc": 0.3103448275862069,
"acc_stderr": 0.049887188500387446,
"em_relaxed": 0.3103448275862069,
"em_relaxed_stderr": 0.049887188500387446
},
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
"acc": 0.24,
"acc_stderr": 0.04292346959909284,
"em_relaxed": 0.24,
"em_relaxed_stderr": 0.04292346959909284
},
"e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": {
"acc": 0.034482758620689655,
"acc_stderr": 0.034482758620689676
},
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
"acc": 0.29,
"acc_stderr": 0.045604802157206845,
"em_relaxed": 0.29,
"em_relaxed_stderr": 0.045604802157206845
},
"ifeval_sk.ifeval|0": {
"inst_level_loose_acc": 0.21307506053268765,
"inst_level_loose_acc_stderr": 0.00041218013349389904,
"inst_level_strict_acc": 0.21307506053268765,
"inst_level_strict_acc_stderr": 0.00041218013349389904,
"prompt_level_loose_acc": 0.10242085661080075,
"prompt_level_loose_acc_stderr": 0.01309628698247887,
"prompt_level_strict_acc": 0.10242085661080075,
"prompt_level_strict_acc_stderr": 0.01309628698247887
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
"acc": 0.18181818181818182,
"acc_stderr": 0.027479603010538804,
"em_relaxed": 0.18181818181818182,
"em_relaxed_stderr": 0.027479603010538804
},
"e_tests.prompt1_qa_sk:geografia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
"acc": 0.22429906542056074,
"acc_stderr": 0.015251413306651283,
"em_relaxed": 0.22429906542056074,
"em_relaxed_stderr": 0.015251413306651283
},
"e_tests.prompt3_qa_sk:slovensky-jazyk|0": {
"qem": 0.0051813471502590676,
"qem_stderr": 0.00518134715025908,
"qf1": 0.008737173625927054,
"qf1_stderr": 0.005740091786106941,
"rouge1": 0.016785287251608494,
"rouge1_stderr": 0.006781696658405232,
"rouge2": 0.00659444182760245,
"rouge2_stderr": 0.005363480846922639,
"rougeL": 0.016785287251608494,
"rougeL_stderr": 0.006781696658405232
},
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
"acc": 0.26495726495726496,
"acc_stderr": 0.028911208802749472,
"em_relaxed": 0.26495726495726496,
"em_relaxed_stderr": 0.028911208802749472
},
"driving_test.prompt1_mqa_single_correct_sk|0": {
"acc": 0.3666191155492154,
"acc_stderr": 0.01821339929295096,
"em_relaxed": 0.3666191155492154,
"em_relaxed_stderr": 0.01821339929295096
},
"scio.prompt1_mqa_single_correct_sk:aj|0": {
"acc": 0.2,
"acc_stderr": 0.13333333333333333,
"em_relaxed": 0.2,
"em_relaxed_stderr": 0.13333333333333333
},
"sklep.prompt2_nli_binary_sk:rte|0": {
"acc": 0.5132530120481927,
"acc_stderr": 0.012271402375351256,
"em_norm_relaxed": 0.5132530120481927,
"em_norm_relaxed_stderr": 0.012271402375351256
},
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
"acc": 0.23857868020304568,
"acc_stderr": 0.03044390069569734,
"em_relaxed": 0.23857868020304568,
"em_relaxed_stderr": 0.03044390069569734
},
"multi_wiki_qa.prompt2_qa_sk|0": {
"qem": 0.08204922953772263,
"qem_stderr": 0.003882716972284686,
"qf1": 0.12697709233547141,
"qf1_stderr": 0.0040052643032774445
},
"e_tests.prompt2_mqa_single_correct_sk:geografia|0": {
"acc": 0.22448979591836735,
"acc_stderr": 0.06022425581505364,
"em_relaxed": 0.22448979591836735,
"em_relaxed_stderr": 0.06022425581505364
},
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
"acc": 0.26,
"acc_stderr": 0.0440844002276808,
"em_relaxed": 0.26,
"em_relaxed_stderr": 0.0440844002276808
},
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
"acc": 0.22442244224422442,
"acc_stderr": 0.024007242951044422,
"em_relaxed": 0.2376237623762376,
"em_relaxed_stderr": 0.0244921064409582
},
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
"acc": 0.2,
"acc_stderr": 0.04020151261036845,
"em_relaxed": 0.2,
"em_relaxed_stderr": 0.04020151261036845
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
"acc": 0.21705426356589147,
"acc_stderr": 0.020982456910832083,
"em_relaxed": 0.21705426356589147,
"em_relaxed_stderr": 0.020982456910832083
},
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
"acc": 0.2875,
"acc_stderr": 0.022658174179374095,
"em_relaxed": 0.2925,
"em_relaxed_stderr": 0.022774020616430368
},
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
"acc": 0.23166023166023167,
"acc_stderr": 0.02626591571601146,
"em_relaxed": 0.23166023166023167,
"em_relaxed_stderr": 0.02626591571601146
},
"e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": {
"acc": 0.034482758620689655,
"acc_stderr": 0.034482758620689676
},
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
"acc": 0.2413793103448276,
"acc_stderr": 0.02399654973117778,
"em_relaxed": 0.2413793103448276,
"em_relaxed_stderr": 0.02399654973117778
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
"acc": 0.18666666666666668,
"acc_stderr": 0.0319208566416966,
"em_relaxed": 0.18666666666666668,
"em_relaxed_stderr": 0.0319208566416966
},
"sklep.prompt3_nli_3_classes_sk:nli|0": {
"acc": 0.002398081534772182,
"acc_stderr": 0.0006915050545011868,
"em_norm_relaxed": 0.002398081534772182,
"em_norm_relaxed_stderr": 0.0006915050545011868
},
"e_tests.prompt3_qa_sk:fyzika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.014285714285714287,
"rouge1_stderr": 0.014285714285714287,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.014285714285714287,
"rougeL_stderr": 0.014285714285714287
},
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
"acc": 0.25892857142857145,
"acc_stderr": 0.041577515398656284,
"em_relaxed": 0.25892857142857145,
"em_relaxed_stderr": 0.041577515398656284
},
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
"acc": 0.3048780487804878,
"acc_stderr": 0.03605784583600454,
"em_relaxed": 0.3048780487804878,
"em_relaxed_stderr": 0.03605784583600454
},
"e_tests.prompt3_mqa_single_correct_sk:informatika|0": {
"acc": 0.224,
"acc_stderr": 0.03744070581082714,
"em_relaxed": 0.224,
"em_relaxed_stderr": 0.03744070581082714
},
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
"acc": 0.28498074454428757,
"acc_stderr": 0.016183663085909714,
"em_relaxed": 0.28498074454428757,
"em_relaxed_stderr": 0.016183663085909714
},
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
"acc": 0.25838926174496646,
"acc_stderr": 0.01794596235855172,
"em_relaxed": 0.26174496644295303,
"em_relaxed_stderr": 0.01802120781376133
},
"arc.prompt3_mqa_single_correct_sk|0": {
"acc": 0.24550898203592814,
"acc_stderr": 0.012593293013246194,
"em_relaxed": 0.24550898203592814,
"em_relaxed_stderr": 0.012593293013246194
},
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
"acc": 0.3191489361702128,
"acc_stderr": 0.030472973363380035,
"em_relaxed": 0.3191489361702128,
"em_relaxed_stderr": 0.030472973363380035
},
"scio.prompt1_mqa_single_correct_sk:vsp|0": {
"acc": 0.24514563106796117,
"acc_stderr": 0.014994906005686719,
"em_relaxed": 0.24514563106796117,
"em_relaxed_stderr": 0.014994906005686719
},
"e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": {
"acc": 0.03571428571428571,
"acc_stderr": 0.025023180348048964
},
"e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": {
"acc": 0.27074235807860264,
"acc_stderr": 0.02078550104939707,
"em_relaxed": 0.27074235807860264,
"em_relaxed_stderr": 0.02078550104939707
},
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
"acc": 0.2631578947368421,
"acc_stderr": 0.03377310252209194,
"em_relaxed": 0.2631578947368421,
"em_relaxed_stderr": 0.03377310252209194
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
"acc": 0.26262626262626265,
"acc_stderr": 0.04445287676983945,
"em_relaxed": 0.26262626262626265,
"em_relaxed_stderr": 0.04445287676983945
},
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
"acc": 0.26785714285714285,
"acc_stderr": 0.04203277291467762,
"em_relaxed": 0.2857142857142857,
"em_relaxed_stderr": 0.04287858751340456
},
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
"acc": 0.16216216216216217,
"acc_stderr": 0.02479468591119148,
"em_relaxed": 0.16666666666666666,
"em_relaxed_stderr": 0.02506903489984114
},
"belebele.prompt1_mqa_single_correct_sk|0": {
"acc": 0.2577777777777778,
"acc_stderr": 0.014588474089651667,
"em_relaxed": 0.2577777777777778,
"em_relaxed_stderr": 0.014588474089651667
},
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
"acc": 0.26785714285714285,
"acc_stderr": 0.04203277291467762,
"em_relaxed": 0.2767857142857143,
"em_relaxed_stderr": 0.042466243366976256
},
"slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.024691358024691357,
"qf1_stderr": 0.024691358024691353,
"rouge1": 0.01851851851851852,
"rouge1_stderr": 0.01851851851851852,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.01851851851851852,
"rougeL_stderr": 0.01851851851851852
},
"movie_review.prompt2_sentiment_binary_sk|0": {
"acc": 0.13097609561752988,
"acc_stderr": 0.007530749388005803,
"em_norm_relaxed": 0.13645418326693226,
"em_norm_relaxed_stderr": 0.007662357895460915
},
"e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": {
"acc": 0.13043478260869565,
"acc_stderr": 0.04084071463598805,
"em_relaxed": 0.14492753623188406,
"em_relaxed_stderr": 0.042689636687312106
},
"e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
"acc": 0.20930232558139536,
"acc_stderr": 0.03110958390976464,
"em_relaxed": 0.20930232558139536,
"em_relaxed_stderr": 0.03110958390976464
},
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
"acc": 0.2277992277992278,
"acc_stderr": 0.026111473881583542,
"em_relaxed": 0.2277992277992278,
"em_relaxed_stderr": 0.026111473881583542
},
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
"acc": 0.2734274711168164,
"acc_stderr": 0.015979778720725094,
"em_relaxed": 0.2734274711168164,
"em_relaxed_stderr": 0.015979778720725094
},
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
"acc": 0.22784810126582278,
"acc_stderr": 0.033475282640609255,
"em_relaxed": 0.22784810126582278,
"em_relaxed_stderr": 0.033475282640609255
},
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
"acc": 0.29906542056074764,
"acc_stderr": 0.044470182376718334,
"em_relaxed": 0.29906542056074764,
"em_relaxed_stderr": 0.044470182376718334
},
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
"acc": 0.21568627450980393,
"acc_stderr": 0.04092563958237656,
"em_relaxed": 0.21568627450980393,
"em_relaxed_stderr": 0.04092563958237656
},
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
"acc": 0.2064343163538874,
"acc_stderr": 0.020985093876713833,
"em_relaxed": 0.2064343163538874,
"em_relaxed_stderr": 0.020985093876713833
},
"slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": {
"acc": 0.25203252032520324,
"acc_stderr": 0.022633188466788714,
"em_relaxed": 0.25203252032520324,
"em_relaxed_stderr": 0.022633188466788714
},
"e_tests.prompt3_qa_sk:dejepis|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"scio.prompt1_mqa_single_correct_sk:mat|0": {
"acc": 0.1885245901639344,
"acc_stderr": 0.020472706124868213,
"em_relaxed": 0.1912568306010929,
"em_relaxed_stderr": 0.020585781457546286
},
"e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": {
"acc": 0.02857142857142857,
"acc_stderr": 0.028571428571428574
},
"scio.prompt3_mqa_single_correct_sk:mat|0": {
"acc": 0.31420765027322406,
"acc_stderr": 0.024297313836707593,
"em_relaxed": 0.31693989071038253,
"em_relaxed_stderr": 0.024354066150157113
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
"acc": 0.22340425531914893,
"acc_stderr": 0.024847921358063962,
"em_relaxed": 0.23404255319148937,
"em_relaxed_stderr": 0.025257861359432414
},
"slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": {
"acc": 0.14285714285714285,
"acc_stderr": 0.03431317581537582,
"em_relaxed": 0.14285714285714285,
"em_relaxed_stderr": 0.03431317581537582
},
"slovakcopa.prompt2_copa_sk|0": {
"acc": 0.392,
"acc_stderr": 0.021854684955611256,
"em_relaxed": 0.394,
"em_relaxed_stderr": 0.021874299301689253
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
"acc": 0.22962962962962963,
"acc_stderr": 0.036333844140734636,
"em_relaxed": 0.22962962962962963,
"em_relaxed_stderr": 0.036333844140734636
},
"e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": {
"acc": 0.02857142857142857,
"acc_stderr": 0.014130710990583526
},
"belebele.prompt2_mqa_single_correct_sk|0": {
"acc": 0.27111111111111114,
"acc_stderr": 0.014826016446581962,
"em_relaxed": 0.27111111111111114,
"em_relaxed_stderr": 0.014826016446581962
},
"skj1.prompt1_skj1_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_relaxed": 0.01,
"em_relaxed_stderr": 0.009999999999999997
},
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
"acc": 0.26,
"acc_stderr": 0.0440844002276808,
"em_relaxed": 0.26,
"em_relaxed_stderr": 0.0440844002276808
},
"driving_test.prompt2_mqa_single_correct_sk|0": {
"acc": 0.2738944365192582,
"acc_stderr": 0.01685553212123784,
"em_relaxed": 0.2738944365192582,
"em_relaxed_stderr": 0.01685553212123784
},
"scio.prompt3_mqa_single_correct_sk:aj|0": {
"acc": 0.3,
"acc_stderr": 0.15275252316519464,
"em_relaxed": 0.3,
"em_relaxed_stderr": 0.15275252316519464
},
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
"acc": 0.26,
"acc_stderr": 0.04408440022768078,
"em_relaxed": 0.26,
"em_relaxed_stderr": 0.04408440022768078
},
"scio.prompt1_mqa_single_correct_sk:bio|0": {
"acc": 0.21238938053097345,
"acc_stderr": 0.022246609446382862,
"em_relaxed": 0.21238938053097345,
"em_relaxed_stderr": 0.022246609446382862
},
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
"acc": 0.24,
"acc_stderr": 0.04292346959909283,
"em_relaxed": 0.24,
"em_relaxed_stderr": 0.04292346959909283
},
"skj1.prompt3_skj1_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_relaxed": 0.0,
"em_relaxed_stderr": 0.0
},
"skj1.prompt2_skj1_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_relaxed": 0.0,
"em_relaxed_stderr": 0.0
},
"e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": {
"acc": 0.02857142857142857,
"acc_stderr": 0.028571428571428574
},
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
"acc": 0.22033898305084745,
"acc_stderr": 0.03831824849223319,
"em_relaxed": 0.22033898305084745,
"em_relaxed_stderr": 0.03831824849223319
},
"e_tests.prompt1_mqa_single_correct_sk:dejepis|0": {
"acc": 0.1875,
"acc_stderr": 0.05693291124629694,
"em_relaxed": 0.1875,
"em_relaxed_stderr": 0.05693291124629694
},
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
"acc": 0.23391812865497075,
"acc_stderr": 0.03246721765117826,
"em_relaxed": 0.23391812865497075,
"em_relaxed_stderr": 0.03246721765117826
},
"slovak_tests.prompt1_qa_sk:maturita|0": {
"qem": 0.008849557522123894,
"qem_stderr": 0.0062436609110668876,
"qf1": 0.007079646017699116,
"qf1_stderr": 0.0038256704785137783,
"rouge1": 0.014790713408591503,
"rouge1_stderr": 0.004864012429457399,
"rouge2": 0.003448518050287962,
"rouge2_stderr": 0.002428770392002376,
"rougeL": 0.014790713408591503,
"rougeL_stderr": 0.004864012429457399
},
"sklep.prompt1_qa_sk:question-answering|0": {
"qem": 0.05248878221851195,
"qem_stderr": 0.002278228840555749,
"qf1": 0.1424834912487429,
"qf1_stderr": 0.0027244148341408567
},
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
"acc": 0.2725,
"acc_stderr": 0.022290166077429768,
"em_relaxed": 0.2725,
"em_relaxed_stderr": 0.022290166077429768
},
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
"acc": 0.24846625766871167,
"acc_stderr": 0.02396990269668754,
"em_relaxed": 0.24846625766871167,
"em_relaxed_stderr": 0.02396990269668754
},
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
"acc": 0.18468468468468469,
"acc_stderr": 0.026102500004017648,
"em_relaxed": 0.18468468468468469,
"em_relaxed_stderr": 0.026102500004017648
},
"e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": {
"acc": 0.03571428571428571,
"acc_stderr": 0.03571428571428572
},
"sklep.prompt1_nli_3_classes_sk:nli|0": {
"acc": 0.004396482813749001,
"acc_stderr": 0.0009353639970831211,
"em_norm_relaxed": 0.004996003197442046,
"em_norm_relaxed_stderr": 0.0009968011147772275
},
"e_tests.prompt2_qa_sk:obcianska-nauka|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.020408163265306124,
"rouge1_stderr": 0.020408163265306124,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.020408163265306124,
"rougeL_stderr": 0.020408163265306124
},
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
"acc": 0.24358974358974358,
"acc_stderr": 0.028120966503914394,
"em_relaxed": 0.24358974358974358,
"em_relaxed_stderr": 0.028120966503914394
},
"financial_exams.prompt3_completion_sk|0": {
"acc": 0.18590704647676162,
"acc_stderr": 0.010655402235931219,
"em_relaxed": 0.18590704647676162,
"em_relaxed_stderr": 0.010655402235931219
},
"arc.prompt1_mqa_single_correct_sk|0": {
"acc": 0.24721984602224123,
"acc_stderr": 0.012622759999132588,
"em_relaxed": 0.24721984602224123,
"em_relaxed_stderr": 0.012622759999132588
},
"slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": {
"acc": 0.30414746543778803,
"acc_stderr": 0.0313021257639245,
"em_relaxed": 0.30414746543778803,
"em_relaxed_stderr": 0.0313021257639245
},
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
"acc": 0.09213051823416507,
"acc_stderr": 0.00896371626415267,
"em_norm_relaxed": 0.09404990403071017,
"em_norm_relaxed_stderr": 0.009047028350861394
},
"e_tests.prompt2_mqa_single_correct_sk:matematika|0": {
"acc": 0.15384615384615385,
"acc_stderr": 0.05852968540405248,
"em_relaxed": 0.15384615384615385,
"em_relaxed_stderr": 0.05852968540405248
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
"acc": 0.20567375886524822,
"acc_stderr": 0.024112138950471887,
"em_relaxed": 0.20567375886524822,
"em_relaxed_stderr": 0.024112138950471887
},
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
"acc": 0.2601880877742947,
"acc_stderr": 0.024603155110002158,
"em_relaxed": 0.26332288401253917,
"em_relaxed_stderr": 0.02469842907233297
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
"acc": 0.1822429906542056,
"acc_stderr": 0.026451344562470145,
"em_relaxed": 0.1822429906542056,
"em_relaxed_stderr": 0.026451344562470145
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
"acc": 0.22807017543859648,
"acc_stderr": 0.056069817847611766,
"em_relaxed": 0.22807017543859648,
"em_relaxed_stderr": 0.056069817847611766
},
"e_tests.prompt1_qa_sk:fyzika|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.014285714285714287,
"rouge1_stderr": 0.014285714285714287,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.014285714285714287,
"rougeL_stderr": 0.014285714285714287
},
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
"acc": 0.24242424242424243,
"acc_stderr": 0.04329004329004326,
"em_relaxed": 0.24242424242424243,
"em_relaxed_stderr": 0.04329004329004326
},
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
"acc": 0.19,
"acc_stderr": 0.03942772444036623,
"em_relaxed": 0.19,
"em_relaxed_stderr": 0.03942772444036623
},
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
"acc": 0.2675,
"acc_stderr": 0.022160484939004153,
"em_relaxed": 0.27,
"em_relaxed_stderr": 0.022225772481648733
},
"e_tests.prompt2_qa_sk:slovensky-jazyk|0": {
"qem": 0.0051813471502590676,
"qem_stderr": 0.00518134715025908,
"qf1": 0.010905502097211939,
"qf1_stderr": 0.0062654994299230354,
"rouge1": 0.015304902351534477,
"rouge1_stderr": 0.006747971135704163,
"rouge2": 0.0072538860103626935,
"rouge2_stderr": 0.005570450232424934,
"rougeL": 0.015304902351534477,
"rougeL_stderr": 0.006747971135704163
},
"slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": {
"acc": 0.30414746543778803,
"acc_stderr": 0.0313021257639245,
"em_relaxed": 0.30414746543778803,
"em_relaxed_stderr": 0.0313021257639245
},
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
"acc": 0.3515981735159817,
"acc_stderr": 0.03233829906905826,
"em_relaxed": 0.3515981735159817,
"em_relaxed_stderr": 0.03233829906905826
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
"acc": 0.2706766917293233,
"acc_stderr": 0.027293707912941223,
"em_relaxed": 0.2706766917293233,
"em_relaxed_stderr": 0.027293707912941223
},
"e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
"acc": 0.27522935779816515,
"acc_stderr": 0.042976970405367555,
"em_relaxed": 0.28440366972477066,
"em_relaxed_stderr": 0.043409998674551635
},
"slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": {
"acc": 0.2791327913279133,
"acc_stderr": 0.02338348056654487,
"em_relaxed": 0.2791327913279133,
"em_relaxed_stderr": 0.02338348056654487
},
"slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": {
"acc": 0.26666666666666666,
"acc_stderr": 0.04336290903919941,
"em_relaxed": 0.26666666666666666,
"em_relaxed_stderr": 0.04336290903919941
},
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
"acc": 0.1513157894736842,
"acc_stderr": 0.029162631596843982,
"em_relaxed": 0.1513157894736842,
"em_relaxed_stderr": 0.029162631596843982
},
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
"acc": 0.2828282828282828,
"acc_stderr": 0.04549461117008498,
"em_relaxed": 0.2828282828282828,
"em_relaxed_stderr": 0.04549461117008498
},
"e_tests.prompt3_mqa_single_correct_sk:biologia|0": {
"acc": 0.22988505747126436,
"acc_stderr": 0.045371581852507746,
"em_relaxed": 0.22988505747126436,
"em_relaxed_stderr": 0.045371581852507746
},
"e_tests.prompt1_qa_sk:chemia|0": {
"qem": 0.018518518518518517,
"qem_stderr": 0.018518518518518514,
"qf1": 0.012345679012345678,
"qf1_stderr": 0.012345679012345675,
"rouge1": 0.02580246913580247,
"rouge1_stderr": 0.011283149325161083,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.02580246913580247,
"rougeL_stderr": 0.011283149325161083
},
"e_tests.prompt2_qa_sk:biologia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.019047619047619046,
"qf1_stderr": 0.019047619047619035,
"rouge1": 0.02857142857142857,
"rouge1_stderr": 0.01990372988626389,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.02857142857142857,
"rougeL_stderr": 0.01990372988626389
},
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
"acc": 0.3,
"acc_stderr": 0.046056618647183814,
"em_relaxed": 0.3,
"em_relaxed_stderr": 0.046056618647183814
},
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
"acc": 0.27,
"acc_stderr": 0.044619604333847394,
"em_relaxed": 0.27,
"em_relaxed_stderr": 0.044619604333847394
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
"acc": 0.2,
"acc_stderr": 0.032769276820761584,
"em_relaxed": 0.20666666666666667,
"em_relaxed_stderr": 0.033171868444173455
},
"slovak_tests.prompt2_qa_sk:monitor9|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"scio.prompt2_mqa_single_correct_sk:mat|0": {
"acc": 0.23770491803278687,
"acc_stderr": 0.022280975973791882,
"em_relaxed": 0.24316939890710382,
"em_relaxed_stderr": 0.022454705846132374
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
"acc": 0.18686868686868688,
"acc_stderr": 0.027772533334218974,
"em_relaxed": 0.1919191919191919,
"em_relaxed_stderr": 0.028057791672989024
},
"multi_wiki_qa.prompt1_qa_sk|0": {
"qem": 0.0756453872323394,
"qem_stderr": 0.003741099997048089,
"qf1": 0.12194744826062183,
"qf1_stderr": 0.0038908205119327767
},
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
"acc": 0.29,
"acc_stderr": 0.045604802157206845,
"em_relaxed": 0.29,
"em_relaxed_stderr": 0.045604802157206845
},
"e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": {
"acc": 0.03571428571428571,
"acc_stderr": 0.03571428571428572
},
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
"acc": 0.18018018018018017,
"acc_stderr": 0.02585333527853839,
"em_relaxed": 0.18018018018018017,
"em_relaxed_stderr": 0.02585333527853839
},
"scio.prompt2_mqa_single_correct_sk:vsp|0": {
"acc": 0.25849514563106796,
"acc_stderr": 0.015261010021381638,
"em_relaxed": 0.2621359223300971,
"em_relaxed_stderr": 0.01533033114193304
},
"e_tests.prompt2_qa_sk:dejepis|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
"acc": 0.16580310880829016,
"acc_stderr": 0.026839845022314415,
"em_relaxed": 0.16580310880829016,
"em_relaxed_stderr": 0.026839845022314415
},
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
"acc": 0.21,
"acc_stderr": 0.040936018074033256,
"em_relaxed": 0.21,
"em_relaxed_stderr": 0.040936018074033256
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
"acc": 0.23232323232323232,
"acc_stderr": 0.04266016017054689,
"em_relaxed": 0.24242424242424243,
"em_relaxed_stderr": 0.04329004329004326
},
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
"acc": 0.2463768115942029,
"acc_stderr": 0.025984280443996907,
"em_relaxed": 0.2463768115942029,
"em_relaxed_stderr": 0.025984280443996907
},
"e_tests.prompt1_qa_sk:informatika|0": {
"qem": 0.038461538461538464,
"qem_stderr": 0.021915495473007307,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0032865282865282863,
"rouge1_stderr": 0.0022973226212271926,
"rouge2": 0.0008547008547008547,
"rouge2_stderr": 0.0008547008547008546,
"rougeL": 0.0032865282865282863,
"rougeL_stderr": 0.0022973226212271926
},
"e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
"acc": 0.18,
"acc_stderr": 0.038612291966536955,
"em_relaxed": 0.18,
"em_relaxed_stderr": 0.038612291966536955
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
"acc": 0.22119815668202766,
"acc_stderr": 0.02824082382619082,
"em_relaxed": 0.2304147465437788,
"em_relaxed_stderr": 0.0286521112206854
},
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
"acc": 0.22162883845126835,
"acc_stderr": 0.015186430423507605,
"em_relaxed": 0.22162883845126835,
"em_relaxed_stderr": 0.015186430423507605
},
"slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": {
"acc": 0.24744897959183673,
"acc_stderr": 0.021823400829422215,
"em_relaxed": 0.24744897959183673,
"em_relaxed_stderr": 0.021823400829422215
},
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
"acc": 0.3,
"acc_stderr": 0.046056618647183814,
"em_relaxed": 0.3,
"em_relaxed_stderr": 0.046056618647183814
},
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
"acc": 0.397172236503856,
"acc_stderr": 0.01755397820008727,
"em_relaxed": 0.40359897172236503,
"em_relaxed_stderr": 0.017600852681572653
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
"acc": 0.24561403508771928,
"acc_stderr": 0.057521390564052664,
"em_relaxed": 0.24561403508771928,
"em_relaxed_stderr": 0.057521390564052664
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
"acc": 0.21963824289405684,
"acc_stderr": 0.021072124180856616,
"em_relaxed": 0.21963824289405684,
"em_relaxed_stderr": 0.021072124180856616
},
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
"acc": 0.2803738317757009,
"acc_stderr": 0.04362839933570099,
"em_relaxed": 0.2803738317757009,
"em_relaxed_stderr": 0.04362839933570099
},
"e_tests.prompt1_mqa_single_correct_sk:geografia|0": {
"acc": 0.24489795918367346,
"acc_stderr": 0.062069005411206336,
"em_relaxed": 0.24489795918367346,
"em_relaxed_stderr": 0.062069005411206336
},
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
"acc": 0.13624678663239073,
"acc_stderr": 0.012306865740559735,
"em_relaxed": 0.14524421593830333,
"em_relaxed_stderr": 0.012640374409743442
},
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
"acc": 0.21739130434782608,
"acc_stderr": 0.024872937110754584,
"em_relaxed": 0.21739130434782608,
"em_relaxed_stderr": 0.024872937110754584
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
"acc": 0.24444444444444444,
"acc_stderr": 0.03212157057535212,
"em_relaxed": 0.24444444444444444,
"em_relaxed_stderr": 0.03212157057535212
},
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
"acc": 0.22830440587449932,
"acc_stderr": 0.015347207245326627,
"em_relaxed": 0.22963951935914553,
"em_relaxed_stderr": 0.015378696047536682
},
"e_tests.prompt3_qa_sk:informatika|0": {
"qem": 0.02564102564102564,
"qem_stderr": 0.01801282495677459,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0036324786324786326,
"rouge1_stderr": 0.0025563439430967027,
"rouge2": 0.0008547008547008547,
"rouge2_stderr": 0.0008547008547008546,
"rougeL": 0.0036324786324786326,
"rougeL_stderr": 0.0025563439430967027
},
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
"acc": 0.21753246753246752,
"acc_stderr": 0.023546481379339165,
"em_relaxed": 0.21753246753246752,
"em_relaxed_stderr": 0.023546481379339165
},
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
"acc": 0.25,
"acc_stderr": 0.04351941398892446,
"em_relaxed": 0.25,
"em_relaxed_stderr": 0.04351941398892446
},
"scio.prompt3_mqa_single_correct_sk:bio|0": {
"acc": 0.24778761061946902,
"acc_stderr": 0.023482897631940607,
"em_relaxed": 0.26548672566371684,
"em_relaxed_stderr": 0.024019440114877274
},
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
"acc": 0.30275229357798167,
"acc_stderr": 0.04421049631437605,
"em_relaxed": 0.30275229357798167,
"em_relaxed_stderr": 0.04421049631437605
},
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
"acc": 0.42287917737789205,
"acc_stderr": 0.017722745992252577,
"em_relaxed": 0.42287917737789205,
"em_relaxed_stderr": 0.017722745992252577
},
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
"acc": 0.17647058823529413,
"acc_stderr": 0.0379328118530781,
"em_relaxed": 0.18627450980392157,
"em_relaxed_stderr": 0.03873958714149352
},
"financial_exams.prompt1_completion_sk|0": {
"acc": 0.18290854572713644,
"acc_stderr": 0.010588568678863373,
"em_relaxed": 0.18290854572713644,
"em_relaxed_stderr": 0.010588568678863373
},
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
"acc": 0.3211009174311927,
"acc_stderr": 0.044927422774799206,
"em_relaxed": 0.3211009174311927,
"em_relaxed_stderr": 0.044927422774799206
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
"acc": 0.20666666666666667,
"acc_stderr": 0.033171868444173455,
"em_relaxed": 0.22,
"em_relaxed_stderr": 0.033936379482039564
},
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
"acc": 0.0932524639878696,
"acc_stderr": 0.00800968883832857,
"em_norm_relaxed": 0.09401061410159212,
"em_norm_relaxed_stderr": 0.008038819818872478
},
"e_tests.prompt3_qa_sk:geografia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
"acc": 0.233502538071066,
"acc_stderr": 0.030218515643241007,
"em_relaxed": 0.233502538071066,
"em_relaxed_stderr": 0.030218515643241007
},
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
"acc": 0.21212121212121213,
"acc_stderr": 0.041296069325408906,
"em_relaxed": 0.21212121212121213,
"em_relaxed_stderr": 0.041296069325408906
},
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
"acc": 0.23232323232323232,
"acc_stderr": 0.042660160170546885,
"em_relaxed": 0.23232323232323232,
"em_relaxed_stderr": 0.042660160170546885
},
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
"acc": 0.26785714285714285,
"acc_stderr": 0.04203277291467762,
"em_relaxed": 0.2767857142857143,
"em_relaxed_stderr": 0.04246624336697624
},
"slovakcopa.prompt3_copa_sk|0": {
"acc": 0.016,
"acc_stderr": 0.0056170367814719815,
"em_relaxed": 0.016,
"em_relaxed_stderr": 0.0056170367814719815
},
"e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": {
"acc": 0.02142857142857143,
"acc_stderr": 0.012282463299882659
},
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
"acc": 0.27102803738317754,
"acc_stderr": 0.043172737765666686,
"em_relaxed": 0.27102803738317754,
"em_relaxed_stderr": 0.043172737765666686
},
"slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": {
"acc": 0.2903225806451613,
"acc_stderr": 0.030884745411494743,
"em_relaxed": 0.2903225806451613,
"em_relaxed_stderr": 0.030884745411494743
},
"financial_exams.prompt2_completion_sk|0": {
"acc": 0.1896551724137931,
"acc_stderr": 0.010737475929667924,
"em_relaxed": 0.1904047976011994,
"em_relaxed_stderr": 0.010753697898553788
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
"acc": 0.19170984455958548,
"acc_stderr": 0.02840895362624528,
"em_relaxed": 0.19170984455958548,
"em_relaxed_stderr": 0.02840895362624528
},
"e_tests.prompt1_qa_sk:slovensky-jazyk|0": {
"qem": 0.0051813471502590676,
"qem_stderr": 0.00518134715025908,
"qf1": 0.01526314790563495,
"qf1_stderr": 0.006730183842906608,
"rouge1": 0.01724455022545386,
"rouge1_stderr": 0.006851608530741241,
"rouge2": 0.0073447868375602215,
"rouge2_stderr": 0.005570575389175516,
"rougeL": 0.01724455022545386,
"rougeL_stderr": 0.006851608530741241
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
"acc": 0.22962962962962963,
"acc_stderr": 0.03633384414073462,
"em_relaxed": 0.22962962962962963,
"em_relaxed_stderr": 0.03633384414073462
},
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
"acc": 0.2862644415917843,
"acc_stderr": 0.01620550507370852,
"em_relaxed": 0.28754813863928114,
"em_relaxed_stderr": 0.01622718713515171
},
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
"acc": 0.2537313432835821,
"acc_stderr": 0.037731914327133756,
"em_relaxed": 0.2537313432835821,
"em_relaxed_stderr": 0.037731914327133756
},
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
"acc": 0.04702495201535509,
"acc_stderr": 0.006561144166938836,
"em_norm_relaxed": 0.05566218809980806,
"em_norm_relaxed_stderr": 0.007105889705516419
},
"toxic_sk.prompt3_toxicity_sk|0": {
"acc": 0.007945516458569807,
"acc_stderr": 0.0029928675111869866,
"em_norm_relaxed": 0.007945516458569807,
"em_norm_relaxed_stderr": 0.0029928675111869866
},
"e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": {
"acc": 0.017857142857142856,
"acc_stderr": 0.017857142857142863
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
"acc": 0.24561403508771928,
"acc_stderr": 0.05752139056405266,
"em_relaxed": 0.24561403508771928,
"em_relaxed_stderr": 0.05752139056405266
},
"slovak_tests.prompt3_qa_sk:maturita|0": {
"qem": 0.01327433628318584,
"qem_stderr": 0.007629803632081178,
"qf1": 0.011971484759095378,
"qf1_stderr": 0.0058019008534905304,
"rouge1": 0.014379696684114138,
"rouge1_stderr": 0.0046830153944730545,
"rouge2": 0.004080629301868239,
"rouge2_stderr": 0.002569082757498883,
"rougeL": 0.014379696684114138,
"rougeL_stderr": 0.0046830153944730545
},
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
"acc": 0.2,
"acc_stderr": 0.04020151261036845,
"em_relaxed": 0.2,
"em_relaxed_stderr": 0.04020151261036845
},
"slovak_tests.prompt2_qa_sk:maturita|0": {
"qem": 0.008849557522123894,
"qem_stderr": 0.0062436609110668876,
"qf1": 0.0057030481809242865,
"qf1_stderr": 0.0035655859886822355,
"rouge1": 0.015324943300924803,
"rouge1_stderr": 0.0047498192758248054,
"rouge2": 0.0022123893805309734,
"rouge2_stderr": 0.002212389380530974,
"rougeL": 0.015324943300924803,
"rougeL_stderr": 0.0047498192758248054
},
"slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.036387264457439894,
"qf1_stderr": 0.025966525279646633,
"rouge1": 0.027777777777777776,
"rouge1_stderr": 0.019641855032959652,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.027777777777777776,
"rougeL_stderr": 0.019641855032959652
},
"e_tests.prompt2_mqa_single_correct_sk:dejepis|0": {
"acc": 0.1875,
"acc_stderr": 0.05693291124629694,
"em_relaxed": 0.1875,
"em_relaxed_stderr": 0.05693291124629694
},
"slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": {
"acc": 0.2653061224489796,
"acc_stderr": 0.022327416188566384,
"em_relaxed": 0.2653061224489796,
"em_relaxed_stderr": 0.022327416188566384
},
"e_tests.prompt2_qa_sk:chemia|0": {
"qem": 0.037037037037037035,
"qem_stderr": 0.025940896147128693,
"qf1": 0.012345679012345678,
"qf1_stderr": 0.012345679012345675,
"rouge1": 0.02580246913580247,
"rouge1_stderr": 0.011283149325161083,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.02580246913580247,
"rougeL_stderr": 0.011283149325161083
},
"e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": {
"acc": 0.26419213973799127,
"acc_stderr": 0.020624529862085718,
"em_relaxed": 0.26419213973799127,
"em_relaxed_stderr": 0.020624529862085718
},
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
"acc": 0.22402597402597402,
"acc_stderr": 0.02379597980086175,
"em_relaxed": 0.22402597402597402,
"em_relaxed_stderr": 0.02379597980086175
},
"e_tests.prompt1_mqa_single_correct_sk:fyzika|0": {
"acc": 0.22727272727272727,
"acc_stderr": 0.0914486154730632,
"em_relaxed": 0.22727272727272727,
"em_relaxed_stderr": 0.0914486154730632
},
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
"acc": 0.19318181818181818,
"acc_stderr": 0.024344064423295675,
"em_relaxed": 0.19318181818181818,
"em_relaxed_stderr": 0.024344064423295675
},
"e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": {
"acc": 0.1,
"acc_stderr": 0.06882472016116853
},
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
"acc": 0.29,
"acc_stderr": 0.045604802157206845,
"em_relaxed": 0.3,
"em_relaxed_stderr": 0.046056618647183814
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
"acc": 0.24812030075187969,
"acc_stderr": 0.02653275776026344,
"em_relaxed": 0.2518796992481203,
"em_relaxed_stderr": 0.026666091559782525
},
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
"acc": 0.2605633802816901,
"acc_stderr": 0.03696558732615874,
"em_relaxed": 0.2605633802816901,
"em_relaxed_stderr": 0.03696558732615874
},
"e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": {
"acc": 0.02857142857142857,
"acc_stderr": 0.028571428571428574
},
"slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.024691358024691357,
"qf1_stderr": 0.024691358024691353,
"rouge1": 0.01851851851851852,
"rouge1_stderr": 0.01851851851851852,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.01851851851851852,
"rougeL_stderr": 0.01851851851851852
},
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
"acc": 0.23134328358208955,
"acc_stderr": 0.036565274697599,
"em_relaxed": 0.23134328358208955,
"em_relaxed_stderr": 0.036565274697599
},
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
"acc": 0.26,
"acc_stderr": 0.04408440022768078,
"em_relaxed": 0.26,
"em_relaxed_stderr": 0.04408440022768078
},
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
"acc": 0.2088607594936709,
"acc_stderr": 0.03244180901572427,
"em_relaxed": 0.2088607594936709,
"em_relaxed_stderr": 0.03244180901572427
},
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
"acc": 0.31,
"acc_stderr": 0.04648231987117316,
"em_relaxed": 0.31,
"em_relaxed_stderr": 0.04648231987117316
},
"scio.prompt2_mqa_single_correct_sk:aj|0": {
"acc": 0.3,
"acc_stderr": 0.15275252316519464,
"em_relaxed": 0.3,
"em_relaxed_stderr": 0.15275252316519464
},
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
"acc": 0.21739130434782608,
"acc_stderr": 0.038631448549506,
"em_relaxed": 0.25217391304347825,
"em_relaxed_stderr": 0.04067222754154718
},
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
"acc": 0.2457627118644068,
"acc_stderr": 0.03980329854920433,
"em_relaxed": 0.2457627118644068,
"em_relaxed_stderr": 0.03980329854920433
},
"e_tests.prompt1_mqa_single_correct_sk:chemia|0": {
"acc": 0.17857142857142858,
"acc_stderr": 0.05164277182008721,
"em_relaxed": 0.17857142857142858,
"em_relaxed_stderr": 0.05164277182008721
},
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
"acc": 0.2386058981233244,
"acc_stderr": 0.022099063590345355,
"em_relaxed": 0.2386058981233244,
"em_relaxed_stderr": 0.022099063590345355
},
"e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": {
"acc": 0.27510917030567683,
"acc_stderr": 0.020889629452003455,
"em_relaxed": 0.27510917030567683,
"em_relaxed_stderr": 0.020889629452003455
},
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
"acc": 0.17763157894736842,
"acc_stderr": 0.03110318238312337,
"em_relaxed": 0.18421052631578946,
"em_relaxed_stderr": 0.031546980450822305
},
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
"acc": 0.3525398028809704,
"acc_stderr": 0.01315990975593033,
"em_norm_relaxed": 0.35329795299469297,
"em_norm_relaxed_stderr": 0.013166337192115686
},
"e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
"acc": 0.21641791044776118,
"acc_stderr": 0.035707794739409195,
"em_relaxed": 0.22388059701492538,
"em_relaxed_stderr": 0.03614487092173135
},
"e_tests.prompt1_mqa_single_correct_sk:matematika|0": {
"acc": 0.1282051282051282,
"acc_stderr": 0.05423355275954149,
"em_relaxed": 0.1282051282051282,
"em_relaxed_stderr": 0.05423355275954149
},
"sklep.prompt3_qa_sk:question-answering|0": {
"qem": 0.05123656475007826,
"qem_stderr": 0.0022523759848186195,
"qf1": 0.1417299858339243,
"qf1_stderr": 0.002706574024592746
},
"e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": {
"acc": 0.022727272727272728,
"acc_stderr": 0.022727272727272728
},
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
"acc": 0.2077922077922078,
"acc_stderr": 0.023156077580247444,
"em_relaxed": 0.2077922077922078,
"em_relaxed_stderr": 0.023156077580247444
},
"toxic_sk.prompt2_toxicity_sk|0": {
"acc": 0.23609534619750283,
"acc_stderr": 0.014316008748210712,
"em_norm_relaxed": 0.23723041997729852,
"em_norm_relaxed_stderr": 0.014339715502704882
},
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
"acc": 0.24770642201834864,
"acc_stderr": 0.041538463936410155,
"em_relaxed": 0.24770642201834864,
"em_relaxed_stderr": 0.041538463936410155
},
"e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": {
"acc": 0.03571428571428571,
"acc_stderr": 0.03571428571428572
},
"senti_sk.prompt3_sentiment_binary_sk|0": {
"acc": 0.7416845493562232,
"acc_stderr": 0.007169768138157722,
"em_norm_relaxed": 0.7475858369098712,
"em_norm_relaxed_stderr": 0.007115537241408355
},
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
"acc": 0.22758620689655173,
"acc_stderr": 0.03493950380131184,
"em_relaxed": 0.22758620689655173,
"em_relaxed_stderr": 0.03493950380131184
},
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
"acc": 0.1856060606060606,
"acc_stderr": 0.023973722216817445,
"em_relaxed": 0.1856060606060606,
"em_relaxed_stderr": 0.023973722216817445
},
"e_tests.prompt2_mqa_single_correct_sk:informatika|0": {
"acc": 0.232,
"acc_stderr": 0.03790650637827825,
"em_relaxed": 0.232,
"em_relaxed_stderr": 0.03790650637827825
},
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
"acc": 0.27,
"acc_stderr": 0.044619604333847394,
"em_relaxed": 0.27,
"em_relaxed_stderr": 0.044619604333847394
},
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
"acc": 0.27,
"acc_stderr": 0.044619604333847394,
"em_relaxed": 0.27,
"em_relaxed_stderr": 0.044619604333847394
},
"toxic_sk.prompt1_toxicity_sk|0": {
"acc": 0.3530079455164586,
"acc_stderr": 0.01611017944424598,
"em_norm_relaxed": 0.35414301929625425,
"em_norm_relaxed_stderr": 0.016121898615772852
},
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
"acc": 0.2651006711409396,
"acc_stderr": 0.018095094525450207,
"em_relaxed": 0.26677852348993286,
"em_relaxed_stderr": 0.018131533555282176
},
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
"acc": 0.23448275862068965,
"acc_stderr": 0.035306258743465914,
"em_relaxed": 0.23448275862068965,
"em_relaxed_stderr": 0.035306258743465914
},
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
"acc": 0.1956027293404094,
"acc_stderr": 0.010926096810556464,
"em_norm_relaxed": 0.1956027293404094,
"em_norm_relaxed_stderr": 0.010926096810556464
},
"scio.prompt3_mqa_single_correct_sk:che|0": {
"acc": 0.24796747967479674,
"acc_stderr": 0.027588788664854164,
"em_relaxed": 0.24796747967479674,
"em_relaxed_stderr": 0.027588788664854164
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
"acc": 0.21717171717171718,
"acc_stderr": 0.029376616484945637,
"em_relaxed": 0.21717171717171718,
"em_relaxed_stderr": 0.029376616484945637
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
"acc": 0.2222222222222222,
"acc_stderr": 0.031073844843659416,
"em_relaxed": 0.2222222222222222,
"em_relaxed_stderr": 0.031073844843659416
},
"e_tests.prompt1_mqa_single_correct_sk:biologia|0": {
"acc": 0.3103448275862069,
"acc_stderr": 0.04988718850038745,
"em_relaxed": 0.3103448275862069,
"em_relaxed_stderr": 0.04988718850038745
},
"e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"e_tests.prompt3_mqa_single_correct_sk:chemia|0": {
"acc": 0.23214285714285715,
"acc_stderr": 0.05692939024000109,
"em_relaxed": 0.23214285714285715,
"em_relaxed_stderr": 0.05692939024000109
},
"e_tests.prompt2_mqa_single_correct_sk:chemia|0": {
"acc": 0.21428571428571427,
"acc_stderr": 0.055328333517248834,
"em_relaxed": 0.21428571428571427,
"em_relaxed_stderr": 0.055328333517248834
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
"acc": 0.17616580310880828,
"acc_stderr": 0.027493504244548047,
"em_relaxed": 0.17616580310880828,
"em_relaxed_stderr": 0.027493504244548047
},
"e_tests.prompt3_mqa_single_correct_sk:matematika|0": {
"acc": 0.20512820512820512,
"acc_stderr": 0.06550424345215436,
"em_relaxed": 0.20512820512820512,
"em_relaxed_stderr": 0.06550424345215436
},
"e_tests.prompt3_qa_sk:obcianska-nauka|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.020408163265306124,
"rouge1_stderr": 0.020408163265306124,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.020408163265306124,
"rougeL_stderr": 0.020408163265306124
},
"e_tests.prompt3_mqa_single_correct_sk:dejepis|0": {
"acc": 0.14583333333333334,
"acc_stderr": 0.05148145136372286,
"em_relaxed": 0.14583333333333334,
"em_relaxed_stderr": 0.05148145136372286
},
"e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": {
"acc": 0.022727272727272728,
"acc_stderr": 0.022727272727272728
},
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
"acc": 0.25704526046114434,
"acc_stderr": 0.012775945430940199,
"em_relaxed": 0.2681468830059778,
"em_relaxed_stderr": 0.012951062793076921
},
"sklep.prompt2_nli_3_classes_sk:nli|0": {
"acc": 0.04936051159072742,
"acc_stderr": 0.003062545148658699,
"em_norm_relaxed": 0.06474820143884892,
"em_norm_relaxed_stderr": 0.003479067694461015
},
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
"acc": 0.2088607594936709,
"acc_stderr": 0.03244180901572427,
"em_relaxed": 0.2088607594936709,
"em_relaxed_stderr": 0.03244180901572427
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
"acc": 0.2332089552238806,
"acc_stderr": 0.01828244336455249,
"em_relaxed": 0.23507462686567165,
"em_relaxed_stderr": 0.01833308365747139
},
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
"acc": 0.26605504587155965,
"acc_stderr": 0.042521210223474465,
"em_relaxed": 0.27522935779816515,
"em_relaxed_stderr": 0.04297697040536756
},
"senti_sk.prompt2_sentiment_3_class_sk|0": {
"acc": 0.3158007269617276,
"acc_stderr": 0.006797678423770271,
"em_norm_relaxed": 0.32050459696386574,
"em_norm_relaxed_stderr": 0.006824536198921628
},
"slovak_tests.prompt1_qa_sk:monitor9|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
"acc": 0.20833333333333334,
"acc_stderr": 0.02504221193381759,
"em_relaxed": 0.20833333333333334,
"em_relaxed_stderr": 0.02504221193381759
},
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
"acc": 0.24107142857142858,
"acc_stderr": 0.04059867246952688,
"em_relaxed": 0.25892857142857145,
"em_relaxed_stderr": 0.041577515398656284
},
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
"acc": 0.24647887323943662,
"acc_stderr": 0.03629342967383697,
"em_relaxed": 0.24647887323943662,
"em_relaxed_stderr": 0.03629342967383697
},
"e_tests.prompt3_qa_sk:biologia|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.019047619047619046,
"qf1_stderr": 0.019047619047619035,
"rouge1": 0.014285714285714285,
"rouge1_stderr": 0.014285714285714287,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.014285714285714285,
"rougeL_stderr": 0.014285714285714287
},
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
"acc": 0.25705329153605017,
"acc_stderr": 0.02450624980268527,
"em_relaxed": 0.26332288401253917,
"em_relaxed_stderr": 0.024698429072332967
},
"e_tests.prompt3_mqa_single_correct_sk:fyzika|0": {
"acc": 0.18181818181818182,
"acc_stderr": 0.08416546361568647,
"em_relaxed": 0.18181818181818182,
"em_relaxed_stderr": 0.08416546361568647
},
"senti_sk.prompt1_sentiment_3_class_sk|0": {
"acc": 0.25486422920675644,
"acc_stderr": 0.006372867628034562,
"em_norm_relaxed": 0.2572161642078255,
"em_norm_relaxed_stderr": 0.00639209321447505
},
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
"acc": 0.3,
"acc_stderr": 0.046056618647183814,
"em_relaxed": 0.3,
"em_relaxed_stderr": 0.046056618647183814
},
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
"acc": 0.16666666666666666,
"acc_stderr": 0.03708284662416542,
"em_relaxed": 0.16666666666666666,
"em_relaxed_stderr": 0.03708284662416542
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
"acc": 0.25806451612903225,
"acc_stderr": 0.029772845386803175,
"em_relaxed": 0.25806451612903225,
"em_relaxed_stderr": 0.029772845386803175
},
"scio.prompt2_mqa_single_correct_sk:bio|0": {
"acc": 0.23008849557522124,
"acc_stderr": 0.02289335836732507,
"em_relaxed": 0.2448377581120944,
"em_relaxed_stderr": 0.02338842519033371
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
"acc": 0.18691588785046728,
"acc_stderr": 0.02671167024582658,
"em_relaxed": 0.19158878504672897,
"em_relaxed_stderr": 0.026965681762486853
},
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
"acc": 0.2711864406779661,
"acc_stderr": 0.041100705493392085,
"em_relaxed": 0.2711864406779661,
"em_relaxed_stderr": 0.041100705493392085
},
"e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": {
"acc": 0.18840579710144928,
"acc_stderr": 0.047420064740574225,
"em_relaxed": 0.18840579710144928,
"em_relaxed_stderr": 0.047420064740574225
},
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
"acc": 0.24092409240924093,
"acc_stderr": 0.0246081660183802,
"em_relaxed": 0.24752475247524752,
"em_relaxed_stderr": 0.0248343018351657
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
"acc": 0.2727272727272727,
"acc_stderr": 0.03173071239071724,
"em_relaxed": 0.2727272727272727,
"em_relaxed_stderr": 0.03173071239071724
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
"acc": 0.23255813953488372,
"acc_stderr": 0.021502792400356758,
"em_relaxed": 0.23255813953488372,
"em_relaxed_stderr": 0.021502792400356758
},
"scio.prompt1_mqa_single_correct_sk:che|0": {
"acc": 0.27235772357723576,
"acc_stderr": 0.028441059280558723,
"em_relaxed": 0.27235772357723576,
"em_relaxed_stderr": 0.028441059280558723
},
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
"acc": 0.18604651162790697,
"acc_stderr": 0.029758600618213775,
"em_relaxed": 0.18604651162790697,
"em_relaxed_stderr": 0.029758600618213775
},
"slovak_tests.prompt3_qa_sk:monitor9|0": {
"qem": 0.0,
"qem_stderr": 0.0,
"qf1": 0.0,
"qf1_stderr": 0.0,
"rouge1": 0.0,
"rouge1_stderr": 0.0,
"rouge2": 0.0,
"rouge2_stderr": 0.0,
"rougeL": 0.0,
"rougeL_stderr": 0.0
},
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
"acc": 0.21481481481481482,
"acc_stderr": 0.03547854198560827,
"em_relaxed": 0.21481481481481482,
"em_relaxed_stderr": 0.03547854198560827
},
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
"acc": 0.19696969696969696,
"acc_stderr": 0.028335609732463348,
"em_relaxed": 0.19696969696969696,
"em_relaxed_stderr": 0.028335609732463348
},
"e_tests.prompt1_mqa_single_correct_sk:informatika|0": {
"acc": 0.24,
"acc_stderr": 0.03835319728439192,
"em_relaxed": 0.24,
"em_relaxed_stderr": 0.03835319728439192
},
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
"acc": 0.28085106382978725,
"acc_stderr": 0.029379170464124825,
"em_relaxed": 0.28085106382978725,
"em_relaxed_stderr": 0.029379170464124825
},
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
"acc": 0.23102310231023102,
"acc_stderr": 0.02425386079112883,
"em_relaxed": 0.23102310231023102,
"em_relaxed_stderr": 0.02425386079112883
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
"acc": 0.22695035460992907,
"acc_stderr": 0.024987106365642973,
"em_relaxed": 0.23049645390070922,
"em_relaxed_stderr": 0.025123739226872405
},
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
"acc": 0.23333333333333334,
"acc_stderr": 0.03161296043612007,
"em_relaxed": 0.23333333333333334,
"em_relaxed_stderr": 0.03161296043612007
},
"sklep.prompt1_nli_binary_sk:rte|0": {
"acc": 0.26385542168674697,
"acc_stderr": 0.010820351700856653,
"em_norm_relaxed": 0.26626506024096386,
"em_norm_relaxed_stderr": 0.01085184289579353
},
"slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": {
"acc": 0.22959183673469388,
"acc_stderr": 0.02126915716826347,
"em_relaxed": 0.22959183673469388,
"em_relaxed_stderr": 0.02126915716826347
},
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
"acc": 0.3486238532110092,
"acc_stderr": 0.04585456701913343,
"em_relaxed": 0.3486238532110092,
"em_relaxed_stderr": 0.04585456701913343
},
"slohry.prompt1_inflection_sk:inflection_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_declension_sk:declension_high|0": {
"acc": 0.0748502994011976,
"acc_stderr": 0.014420511342862789,
"em_norm_relaxed": 0.0748502994011976,
"em_norm_relaxed_stderr": 0.014420511342862789
},
"slohry.prompt3_gradation_sk:gradation_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_inflection_sk:inflection_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_gradation_sk:gradation_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_declension_sk:declension_high|0": {
"acc": 0.059880239520958084,
"acc_stderr": 0.013002032231530652,
"em_norm_relaxed": 0.059880239520958084,
"em_norm_relaxed_stderr": 0.013002032231530652
},
"chiby.prompt2_chiby_sk|0": {
"acc": 0.025349650349650348,
"acc_stderr": 0.0026834904197041206,
"em_norm_relaxed": 0.02564102564102564,
"em_norm_relaxed_stderr": 0.002698465265697882
},
"slovak_trivia.prompt3_mqa_single_correct_sk|0": {
"acc": 0.28611898016997167,
"acc_stderr": 0.024088786371538086,
"em_relaxed": 0.28611898016997167,
"em_relaxed_stderr": 0.024088786371538086
},
"slohry.prompt1_inflection_sk:inflection_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_gradation_sk:gradation_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"chiby.prompt3_chiby_sk|0": {
"acc": 0.1622960372960373,
"acc_stderr": 0.00629490070363013,
"em_norm_relaxed": 0.16258741258741258,
"em_norm_relaxed_stderr": 0.00629945304550759
},
"slohry.prompt2_declension_sk:declension_mid|0": {
"acc": 0.05389221556886228,
"acc_stderr": 0.012374031301903447,
"em_norm_relaxed": 0.05389221556886228,
"em_norm_relaxed_stderr": 0.012374031301903447
},
"slohry.prompt2_gradation_sk:gradation_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_gradation_sk:gradation_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_identity_form_sk:identity_form_low|0": {
"acc": 0.005988023952095809,
"acc_stderr": 0.004227809946139441,
"em_norm_relaxed": 0.005988023952095809,
"em_norm_relaxed_stderr": 0.004227809946139441
},
"slohry.prompt3_declension_sk:declension_high|0": {
"acc": 0.0718562874251497,
"acc_stderr": 0.014152002095505148,
"em_norm_relaxed": 0.0718562874251497,
"em_norm_relaxed_stderr": 0.014152002095505148
},
"slohry.prompt3_declension_sk:declension_mid|0": {
"acc": 0.0658682634730539,
"acc_stderr": 0.013593148263003514,
"em_norm_relaxed": 0.0658682634730539,
"em_norm_relaxed_stderr": 0.013593148263003514
},
"slohry.prompt1_identity_form_sk:identity_form_high|0": {
"acc": 0.0029940119760479044,
"acc_stderr": 0.0029940119760478922,
"em_norm_relaxed": 0.0029940119760479044,
"em_norm_relaxed_stderr": 0.0029940119760478922
},
"slohry.prompt3_declension_sk:declension_low|0": {
"acc": 0.04790419161676647,
"acc_stderr": 0.011703209185183546,
"em_norm_relaxed": 0.04790419161676647,
"em_norm_relaxed_stderr": 0.011703209185183546
},
"slohry.prompt1_declension_sk:declension_mid|0": {
"acc": 0.0718562874251497,
"acc_stderr": 0.014152002095505158,
"em_norm_relaxed": 0.0718562874251497,
"em_norm_relaxed_stderr": 0.014152002095505158
},
"slohry.prompt3_inflection_sk:inflection_high|0": {
"acc": 0.008982035928143712,
"acc_stderr": 0.005170184492247012,
"em_norm_relaxed": 0.008982035928143712,
"em_norm_relaxed_stderr": 0.005170184492247012
},
"slohry.prompt2_declension_sk:declension_low|0": {
"acc": 0.029940119760479042,
"acc_stderr": 0.00933907625586894,
"em_norm_relaxed": 0.029940119760479042,
"em_norm_relaxed_stderr": 0.00933907625586894
},
"slohry.prompt3_gradation_sk:gradation_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_identity_form_sk:identity_form_low|0": {
"acc": 0.008982035928143712,
"acc_stderr": 0.005170184492247015,
"em_norm_relaxed": 0.008982035928143712,
"em_norm_relaxed_stderr": 0.005170184492247015
},
"slovak_trivia.prompt1_mqa_single_correct_sk|0": {
"acc": 0.26628895184135976,
"acc_stderr": 0.02355959235547647,
"em_relaxed": 0.26628895184135976,
"em_relaxed_stderr": 0.02355959235547647
},
"analogies_sk.prompt3_analogy_sk|0": {
"acc": 0.0,
"acc_stderr": 0.0
},
"slohry.prompt3_gradation_sk:gradation_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_inflection_sk:inflection_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_inflection_sk:inflection_high|0": {
"acc": 0.0029940119760479044,
"acc_stderr": 0.0029940119760478966,
"em_norm_relaxed": 0.0029940119760479044,
"em_norm_relaxed_stderr": 0.0029940119760478966
},
"slohry.prompt3_inflection_sk:inflection_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_identity_form_sk:identity_form_high|0": {
"acc": 0.005988023952095809,
"acc_stderr": 0.004227809946139438,
"em_norm_relaxed": 0.005988023952095809,
"em_norm_relaxed_stderr": 0.004227809946139438
},
"slohry.prompt2_identity_form_sk:identity_form_mid|0": {
"acc": 0.02694610778443114,
"acc_stderr": 0.008873487700057157,
"em_norm_relaxed": 0.02694610778443114,
"em_norm_relaxed_stderr": 0.008873487700057157
},
"slohry.prompt3_identity_form_sk:identity_form_mid|0": {
"acc": 0.008982035928143712,
"acc_stderr": 0.0051701844922470075,
"em_norm_relaxed": 0.008982035928143712,
"em_norm_relaxed_stderr": 0.0051701844922470075
},
"slohry.prompt3_identity_form_sk:identity_form_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt1_declension_sk:declension_low|0": {
"acc": 0.05089820359281437,
"acc_stderr": 0.012044409374838807,
"em_norm_relaxed": 0.05089820359281437,
"em_norm_relaxed_stderr": 0.012044409374838807
},
"slohry.prompt1_gradation_sk:gradation_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"analogies_sk.prompt2_analogy_sk|0": {
"acc": 0.0007544322897019992,
"acc_stderr": 0.0005333635248765946
},
"analogies_sk.prompt1_analogy_sk|0": {
"acc": 0.001131648434552999,
"acc_stderr": 0.0006531109317700804
},
"slohry.prompt1_gradation_sk:gradation_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt3_identity_form_sk:identity_form_high|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_inflection_sk:inflection_mid|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"slohry.prompt2_inflection_sk:inflection_low|0": {
"acc": 0.0,
"acc_stderr": 0.0,
"em_norm_relaxed": 0.0,
"em_norm_relaxed_stderr": 0.0
},
"chiby.prompt1_chiby_sk|0": {
"acc": 0.23572261072261072,
"acc_stderr": 0.00724629095400738,
"em_norm_relaxed": 0.23892773892773891,
"em_norm_relaxed_stderr": 0.007280075348854293
},
"slovak_trivia.prompt2_mqa_single_correct_sk|0": {
"acc": 0.28328611898017,
"acc_stderr": 0.024016749232030024,
"em_relaxed": 0.28328611898017,
"em_relaxed_stderr": 0.024016749232030024
},
"slohry.prompt1_identity_form_sk:identity_form_mid|0": {
"acc": 0.011976047904191617,
"acc_stderr": 0.005960989845848331,
"em_norm_relaxed": 0.011976047904191617,
"em_norm_relaxed_stderr": 0.005960989845848331
},
"movie_review_SK_AVERAGE|0": {
"em_norm_relaxed": 0.14890438247011953,
"acc": 0.1457503320053121,
"em_norm_relaxed_stderr": 0.007823268108665498,
"acc_stderr": 0.007746689156547597,
"em_norm_relaxed__prompts": {
"1": 0.10109561752988047,
"3": 0.20916334661354583,
"2": 0.13645418326693226
},
"acc__prompts": {
"1": 0.09860557768924302,
"3": 0.20766932270916336,
"2": 0.13097609561752988
}
},
"movie_review_AVERAGE|0": {
"em_norm_relaxed": 0.14890438247011953,
"em_norm_relaxed_stderr": 0.007823268108665498,
"acc": 0.1457503320053121,
"acc_stderr": 0.007746689156547597
},
"e_tests_mqa_SK_AVERAGE|0": {
"em_relaxed": 0.24309199020636585,
"acc": 0.18150087260034903,
"em_relaxed_stderr": 0.03621678600189384,
"acc_stderr": 0.03214832055891622,
"em_relaxed__prompts": {
"2": 0.24239244491080797,
"1": 0.2497376705141658,
"3": 0.23714585519412382
},
"acc__prompts": {
"2": 0.18175018698578907,
"1": 0.1862378459237098,
"3": 0.17651458489154825
}
},
"e_tests_mqa_AVERAGE|0": {
"em_relaxed": 0.24309199020636585,
"em_relaxed_stderr": 0.03621678600189384,
"acc": 0.18150087260034903,
"acc_stderr": 0.03214832055891622
},
"mmlu_SK_AVERAGE|0": {
"em_relaxed": 0.24332669831062115,
"acc": 0.24148930740570612,
"em_relaxed_stderr": 0.0266013460946822,
"acc_stderr": 0.026529494985304103,
"em_relaxed__prompts": {
"2": 0.24628693921298422,
"3": 0.2474353085285561,
"1": 0.23625784719032308
},
"acc__prompts": {
"2": 0.2440667585362119,
"3": 0.24437299035369775,
"1": 0.23602817332720868
}
},
"mmlu_AVERAGE|0": {
"em_relaxed": 0.24332669831062115,
"em_relaxed_stderr": 0.0266013460946822,
"acc": 0.24148930740570612,
"acc_stderr": 0.026529494985304103
},
"belebele_SK_AVERAGE|0": {
"em_relaxed": 0.267037037037037,
"acc": 0.26666666666666666,
"em_relaxed_stderr": 0.014753176443555857,
"acc_stderr": 0.014746835660938531,
"em_relaxed__prompts": {
"3": 0.2722222222222222,
"1": 0.2577777777777778,
"2": 0.27111111111111114
},
"acc__prompts": {
"3": 0.27111111111111114,
"1": 0.2577777777777778,
"2": 0.27111111111111114
}
},
"belebele_AVERAGE|0": {
"em_relaxed": 0.267037037037037,
"em_relaxed_stderr": 0.014753176443555857,
"acc": 0.26666666666666666,
"acc_stderr": 0.014746835660938531
},
"senti_sk_binary_SK_AVERAGE|0": {
"em_norm_relaxed": 0.49356223175965663,
"acc": 0.4873927038626609,
"em_norm_relaxed_stderr": 0.007613926970563059,
"acc_stderr": 0.0076136774621452415,
"em_norm_relaxed__prompts": {
"2": 0.3261802575107296,
"1": 0.4069206008583691,
"3": 0.7475858369098712
},
"acc__prompts": {
"2": 0.32135193133047213,
"1": 0.39914163090128757,
"3": 0.7416845493562232
}
},
"senti_sk_binary_AVERAGE|0": {
"em_norm_relaxed": 0.49356223175965663,
"em_norm_relaxed_stderr": 0.007613926970563059,
"acc": 0.4873927038626609,
"acc_stderr": 0.0076136774621452415
},
"slovak_tests_mqa_SK_AVERAGE|0": {
"em_relaxed": 0.2600800246229609,
"acc": 0.2600800246229609,
"em_relaxed_stderr": 0.02573346982142187,
"acc_stderr": 0.02573346982142187,
"em_relaxed__prompts": {
"2": 0.26685133887349954,
"3": 0.25761772853185594,
"1": 0.25577100646352724
},
"acc__prompts": {
"2": 0.26685133887349954,
"3": 0.25761772853185594,
"1": 0.25577100646352724
}
},
"slovak_tests_mqa_AVERAGE|0": {
"em_relaxed": 0.2600800246229609,
"em_relaxed_stderr": 0.02573346982142187,
"acc": 0.2600800246229609,
"acc_stderr": 0.02573346982142187
},
"e_tests_qa_SK_AVERAGE|0": {
"rouge2": 0.003228194982976972,
"rouge1": 0.014818769352489746,
"rougeL": 0.014818769352489746,
"qf1": 0.007878222192390297,
"qem": 0.012039127163280662,
"rouge2_stderr": 0.002547306056000719,
"rouge1_stderr": 0.008024257279824093,
"rougeL_stderr": 0.008024257279824093,
"qf1_stderr": 0.005529975861330393,
"qem_stderr": 0.00874750521016363,
"rouge2__prompts": {
"3": 0.0030234626171420755,
"2": 0.003310759969902182,
"1": 0.0033503623618866578
},
"rouge1__prompts": {
"3": 0.013608596069092682,
"2": 0.014902802239145354,
"1": 0.015944909749231198
},
"rougeL__prompts": {
"3": 0.013608596069092682,
"2": 0.014902802239145354,
"1": 0.015944909749231198
},
"qf1__prompts": {
"3": 0.005311379630859115,
"2": 0.008663871869289476,
"1": 0.009659415077022301
},
"qem__prompts": {
"3": 0.011286681715575621,
"2": 0.013544018058690745,
"1": 0.011286681715575621
}
},
"e_tests_qa_AVERAGE|0": {
"rouge2": 0.003228194982976972,
"rouge2_stderr": 0.002547306056000719,
"rouge1": 0.014818769352489746,
"rouge1_stderr": 0.008024257279824093,
"rougeL": 0.014818769352489746,
"rougeL_stderr": 0.008024257279824093,
"qf1": 0.007878222192390297,
"qf1_stderr": 0.005529975861330393,
"qem": 0.012039127163280662,
"qem_stderr": 0.00874750521016363
},
"scio_SK_AVERAGE|0": {
"em_relaxed": 0.25434173669467786,
"acc": 0.250046685340803,
"em_relaxed_stderr": 0.02074943266589043,
"acc_stderr": 0.020633005498944187,
"em_relaxed__prompts": {
"3": 0.27675070028011206,
"2": 0.2549019607843137,
"1": 0.23137254901960785
},
"acc__prompts": {
"3": 0.2700280112044818,
"2": 0.24929971988795518,
"1": 0.230812324929972
}
},
"scio_AVERAGE|0": {
"em_relaxed": 0.25434173669467786,
"em_relaxed_stderr": 0.02074943266589043,
"acc": 0.250046685340803,
"acc_stderr": 0.020633005498944187
},
"arc_SK_AVERAGE|0": {
"em_relaxed": 0.24664955802680352,
"acc": 0.2446535500427716,
"em_relaxed_stderr": 0.012612937670503793,
"acc_stderr": 0.012578166445440875,
"em_relaxed__prompts": {
"2": 0.24721984602224123,
"3": 0.24550898203592814,
"1": 0.24721984602224123
},
"acc__prompts": {
"2": 0.24123182207014543,
"3": 0.24550898203592814,
"1": 0.24721984602224123
}
},
"arc_AVERAGE|0": {
"em_relaxed": 0.24664955802680352,
"em_relaxed_stderr": 0.012612937670503793,
"acc": 0.2446535500427716,
"acc_stderr": 0.012578166445440875
},
"sklep_rte_SK_AVERAGE|0": {
"em_norm_relaxed": 0.31506024096385543,
"acc": 0.3140562248995984,
"em_norm_relaxed_stderr": 0.010750305246379226,
"acc_stderr": 0.010735367420948657,
"em_norm_relaxed__prompts": {
"3": 0.16566265060240964,
"2": 0.5132530120481927,
"1": 0.26626506024096386
},
"acc__prompts": {
"3": 0.1650602409638554,
"2": 0.5132530120481927,
"1": 0.26385542168674697
}
},
"sklep_rte_AVERAGE|0": {
"em_norm_relaxed": 0.31506024096385543,
"em_norm_relaxed_stderr": 0.010750305246379226,
"acc": 0.3140562248995984,
"acc_stderr": 0.010735367420948657
},
"driving_test_SK_AVERAGE|0": {
"em_relaxed": 0.2710413694721826,
"acc": 0.2710413694721826,
"em_relaxed_stderr": 0.016450865339756114,
"acc_stderr": 0.016450865339756114,
"em_relaxed__prompts": {
"3": 0.17261055634807418,
"1": 0.3666191155492154,
"2": 0.2738944365192582
},
"acc__prompts": {
"3": 0.17261055634807418,
"1": 0.3666191155492154,
"2": 0.2738944365192582
}
},
"driving_test_AVERAGE|0": {
"em_relaxed": 0.2710413694721826,
"em_relaxed_stderr": 0.016450865339756114,
"acc": 0.2710413694721826,
"acc_stderr": 0.016450865339756114
},
"sklep_sentiment_analysis_SK_AVERAGE|0": {
"em_norm_relaxed": 0.11388355726167626,
"acc": 0.10748560460652591,
"em_norm_relaxed_stderr": 0.009453015726623323,
"acc_stderr": 0.009172256989852929,
"em_norm_relaxed__prompts": {
"2": 0.19193857965451055,
"3": 0.09404990403071017,
"1": 0.05566218809980806
},
"acc__prompts": {
"2": 0.1833013435700576,
"3": 0.09213051823416507,
"1": 0.04702495201535509
}
},
"sklep_sentiment_analysis_AVERAGE|0": {
"em_norm_relaxed": 0.11388355726167626,
"em_norm_relaxed_stderr": 0.009453015726623323,
"acc": 0.10748560460652591,
"acc_stderr": 0.009172256989852929
},
"slovakcopa_SK_AVERAGE|0": {
"em_relaxed": 0.27066666666666667,
"acc": 0.27,
"em_relaxed_stderr": 0.016480088564366616,
"acc_stderr": 0.016473550449007283,
"em_relaxed__prompts": {
"1": 0.402,
"2": 0.394,
"3": 0.016
},
"acc__prompts": {
"1": 0.402,
"2": 0.392,
"3": 0.016
}
},
"slovakcopa_AVERAGE|0": {
"em_relaxed": 0.27066666666666667,
"em_relaxed_stderr": 0.016480088564366616,
"acc": 0.27,
"acc_stderr": 0.016473550449007283
},
"sklep_qa_SK_AVERAGE|0": {
"qf1": 0.14283353603077906,
"qem": 0.05203659257713312,
"qf1_stderr": 0.0027203043184777205,
"qem_stderr": 0.002268897741053689,
"qf1__prompts": {
"2": 0.14428713100967,
"1": 0.1424834912487429,
"3": 0.1417299858339243
},
"qem__prompts": {
"2": 0.05238443076280914,
"1": 0.05248878221851195,
"3": 0.05123656475007826
}
},
"sklep_qa_AVERAGE|0": {
"qf1": 0.14283353603077906,
"qf1_stderr": 0.0027203043184777205,
"qem": 0.05203659257713312,
"qem_stderr": 0.002268897741053689
},
"senti_sk_3class_SK_AVERAGE|0": {
"em_norm_relaxed": 0.38621623547858314,
"acc": 0.3813698239612287,
"em_norm_relaxed_stderr": 0.006810719788072698,
"acc_stderr": 0.006801058501709849,
"em_norm_relaxed__prompts": {
"3": 0.5809279452640581,
"2": 0.32050459696386574,
"1": 0.2572161642078255
},
"acc__prompts": {
"3": 0.573444515715202,
"2": 0.3158007269617276,
"1": 0.25486422920675644
}
},
"senti_sk_3class_AVERAGE|0": {
"em_norm_relaxed": 0.38621623547858314,
"em_norm_relaxed_stderr": 0.006810719788072698,
"acc": 0.3813698239612287,
"acc_stderr": 0.006801058501709849
},
"multi_wiki_qa_SK_AVERAGE|0": {
"qf1": 0.12354806704071784,
"qem": 0.07884730838503101,
"qf1_stderr": 0.003950374051156623,
"qem_stderr": 0.0038122173887514235,
"qf1__prompts": {
"3": 0.12171966052606027,
"2": 0.12697709233547141,
"1": 0.12194744826062183
},
"qem__prompts": {
"3": 0.07884730838503101,
"2": 0.08204922953772263,
"1": 0.0756453872323394
}
},
"multi_wiki_qa_AVERAGE|0": {
"qf1": 0.12354806704071784,
"qf1_stderr": 0.003950374051156623,
"qem": 0.07884730838503101,
"qem_stderr": 0.0038122173887514235
},
"ifeval_sk_SK_AVERAGE|0": {
"prompt_level_strict_acc": 0.10242085661080075,
"prompt_level_loose_acc": 0.10242085661080075,
"inst_level_strict_acc": 0.21307506053268765,
"inst_level_loose_acc": 0.21307506053268765,
"prompt_level_strict_acc_stderr": 0.01309628698247887,
"prompt_level_loose_acc_stderr": 0.01309628698247887,
"inst_level_strict_acc_stderr": 0.00041218013349389904,
"inst_level_loose_acc_stderr": 0.00041218013349389904
},
"ifeval_sk_AVERAGE|0": {
"prompt_level_strict_acc": 0.10242085661080075,
"prompt_level_strict_acc_stderr": 0.01309628698247887,
"prompt_level_loose_acc": 0.10242085661080075,
"prompt_level_loose_acc_stderr": 0.01309628698247887,
"inst_level_strict_acc": 0.21307506053268765,
"inst_level_strict_acc_stderr": 0.00041218013349389904,
"inst_level_loose_acc": 0.21307506053268765,
"inst_level_loose_acc_stderr": 0.00041218013349389904
},
"globalmmlu_lite_SK_AVERAGE|0": {
"em_relaxed": 0.2783333333333333,
"acc": 0.2758333333333333,
"em_relaxed_stderr": 0.022429986391836288,
"acc_stderr": 0.022369608398602674,
"em_relaxed__prompts": {
"3": 0.2925,
"1": 0.2725,
"2": 0.27
},
"acc__prompts": {
"3": 0.2875,
"1": 0.2725,
"2": 0.2675
}
},
"globalmmlu_lite_AVERAGE|0": {
"em_relaxed": 0.2783333333333333,
"em_relaxed_stderr": 0.022429986391836288,
"acc": 0.2758333333333333,
"acc_stderr": 0.022369608398602674
},
"sklep_nli_SK_AVERAGE|0": {
"em_norm_relaxed": 0.024047428723687717,
"acc": 0.0187183586464162,
"em_norm_relaxed_stderr": 0.0017224579545798097,
"acc_stderr": 0.001563138066747669,
"em_norm_relaxed__prompts": {
"3": 0.002398081534772182,
"1": 0.004996003197442046,
"2": 0.06474820143884892
},
"acc__prompts": {
"3": 0.002398081534772182,
"1": 0.004396482813749001,
"2": 0.04936051159072742
}
},
"sklep_nli_AVERAGE|0": {
"em_norm_relaxed": 0.024047428723687717,
"em_norm_relaxed_stderr": 0.0017224579545798097,
"acc": 0.0187183586464162,
"acc_stderr": 0.001563138066747669
},
"slovak_tests_qa_SK_AVERAGE|0": {
"rouge2": 0.0030834556044640076,
"rouge1": 0.0149009568631566,
"rougeL": 0.0149009568631566,
"qf1": 0.008916490572837322,
"qem": 0.00980392156862745,
"rouge2_stderr": 0.00228223362995418,
"rouge1_stderr": 0.00523977237139308,
"rougeL_stderr": 0.00523977237139308,
"qf1_stderr": 0.005125765723300312,
"qem_stderr": 0.006367605535927983,
"rouge2__prompts": {
"1": 0.003274643190609577,
"2": 0.0021008403361344537,
"3": 0.0038748832866479918
},
"rouge1__prompts": {
"1": 0.014745243264740951,
"2": 0.015252537196116269,
"3": 0.014705090128612585
},
"rougeL__prompts": {
"1": 0.014745243264740951,
"2": 0.015252537196116269,
"3": 0.014705090128612585
},
"qf1__prompts": {
"1": 0.00765639589169001,
"2": 0.006349206349206349,
"3": 0.012743869477615606
},
"qem__prompts": {
"1": 0.008403361344537815,
"2": 0.008403361344537815,
"3": 0.012605042016806723
}
},
"slovak_tests_qa_AVERAGE|0": {
"rouge2": 0.0030834556044640076,
"rouge2_stderr": 0.00228223362995418,
"rouge1": 0.0149009568631566,
"rouge1_stderr": 0.00523977237139308,
"rougeL": 0.0149009568631566,
"rougeL_stderr": 0.00523977237139308,
"qf1": 0.008916490572837322,
"qf1_stderr": 0.005125765723300312,
"qem": 0.00980392156862745,
"qem_stderr": 0.006367605535927983
},
"skj1_SK_AVERAGE|0": {
"em_relaxed": 0.0033333333333333335,
"acc": 0.0,
"em_relaxed_stderr": 0.0033333333333333322,
"acc_stderr": 0.0,
"em_relaxed__prompts": {
"1": 0.01,
"3": 0.0,
"2": 0.0
},
"acc__prompts": {
"1": 0.0,
"3": 0.0,
"2": 0.0
}
},
"skj1_AVERAGE|0": {
"em_relaxed": 0.0033333333333333335,
"em_relaxed_stderr": 0.0033333333333333322,
"acc": 0.0,
"acc_stderr": 0.0
},
"financial_exams_SK_AVERAGE|0": {
"em_relaxed": 0.18640679660169915,
"acc": 0.18615692153923039,
"em_relaxed_stderr": 0.010665889604449459,
"acc_stderr": 0.010660482281487506,
"em_relaxed__prompts": {
"3": 0.18590704647676162,
"1": 0.18290854572713644,
"2": 0.1904047976011994
},
"acc__prompts": {
"3": 0.18590704647676162,
"1": 0.18290854572713644,
"2": 0.1896551724137931
}
},
"financial_exams_AVERAGE|0": {
"em_relaxed": 0.18640679660169915,
"em_relaxed_stderr": 0.010665889604449459,
"acc": 0.18615692153923039,
"acc_stderr": 0.010660482281487506
},
"truthful_qa_SK_AVERAGE|0": {
"em_relaxed": 0.32390745501285345,
"acc": 0.31876606683804626,
"em_relaxed_stderr": 0.01598799102785622,
"acc_stderr": 0.015861196644299858,
"em_relaxed__prompts": {
"2": 0.40359897172236503,
"3": 0.14524421593830333,
"1": 0.42287917737789205
},
"acc__prompts": {
"2": 0.397172236503856,
"3": 0.13624678663239073,
"1": 0.42287917737789205
}
},
"truthful_qa_AVERAGE|0": {
"em_relaxed": 0.32390745501285345,
"em_relaxed_stderr": 0.01598799102785622,
"acc": 0.31876606683804626,
"acc_stderr": 0.015861196644299858
},
"sklep_hate_speech_SK_AVERAGE|0": {
"em_norm_relaxed": 0.21430376547889815,
"acc": 0.2137983320697498,
"em_norm_relaxed_stderr": 0.010710417940514876,
"acc_stderr": 0.010698565134938456,
"em_norm_relaxed__prompts": {
"3": 0.09401061410159212,
"1": 0.35329795299469297,
"2": 0.1956027293404094
},
"acc__prompts": {
"3": 0.0932524639878696,
"1": 0.3525398028809704,
"2": 0.1956027293404094
}
},
"sklep_hate_speech_AVERAGE|0": {
"em_norm_relaxed": 0.21430376547889815,
"em_norm_relaxed_stderr": 0.010710417940514876,
"acc": 0.2137983320697498,
"acc_stderr": 0.010698565134938456
},
"toxic_sk_SK_AVERAGE|0": {
"em_norm_relaxed": 0.19977298524404086,
"acc": 0.19901626939084374,
"em_norm_relaxed_stderr": 0.011151493876554907,
"acc_stderr": 0.011139685234547891,
"em_norm_relaxed__prompts": {
"3": 0.007945516458569807,
"2": 0.23723041997729852,
"1": 0.35414301929625425
},
"acc__prompts": {
"3": 0.007945516458569807,
"2": 0.23609534619750283,
"1": 0.3530079455164586
}
},
"toxic_sk_AVERAGE|0": {
"em_norm_relaxed": 0.19977298524404086,
"em_norm_relaxed_stderr": 0.011151493876554907,
"acc": 0.19901626939084374,
"acc_stderr": 0.011139685234547891
},
"slohry_SK_AVERAGE|0": {
"em_norm_relaxed": 0.016983016983016984,
"acc": 0.016983016983016984,
"em_norm_relaxed_stderr": 0.004436902963904142,
"acc_stderr": 0.004436902963904142,
"em_norm_relaxed__prompts": {
"1": 0.018731268731268732,
"3": 0.016983016983016984,
"2": 0.015234765234765234
},
"acc__prompts": {
"1": 0.018731268731268732,
"3": 0.016983016983016984,
"2": 0.015234765234765234
}
},
"slohry_AVERAGE|0": {
"em_norm_relaxed": 0.016983016983016984,
"em_norm_relaxed_stderr": 0.004436902963904142,
"acc": 0.016983016983016984,
"acc_stderr": 0.004436902963904142
},
"chiby_SK_AVERAGE|0": {
"em_norm_relaxed": 0.14238539238539238,
"acc": 0.1411227661227661,
"em_norm_relaxed_stderr": 0.005425997886686588,
"acc_stderr": 0.005408227359113877,
"em_norm_relaxed__prompts": {
"2": 0.02564102564102564,
"3": 0.16258741258741258,
"1": 0.23892773892773891
},
"acc__prompts": {
"2": 0.025349650349650348,
"3": 0.1622960372960373,
"1": 0.23572261072261072
}
},
"chiby_AVERAGE|0": {
"em_norm_relaxed": 0.14238539238539238,
"em_norm_relaxed_stderr": 0.005425997886686588,
"acc": 0.1411227661227661,
"acc_stderr": 0.005408227359113877
},
"slovak_trivia_SK_AVERAGE|0": {
"em_relaxed": 0.2785646836638338,
"acc": 0.2785646836638338,
"em_relaxed_stderr": 0.023888375986348194,
"acc_stderr": 0.023888375986348194,
"em_relaxed__prompts": {
"3": 0.28611898016997167,
"1": 0.26628895184135976,
"2": 0.28328611898017
},
"acc__prompts": {
"3": 0.28611898016997167,
"1": 0.26628895184135976,
"2": 0.28328611898017
}
},
"slovak_trivia_AVERAGE|0": {
"em_relaxed": 0.2785646836638338,
"em_relaxed_stderr": 0.023888375986348194,
"acc": 0.2785646836638338,
"acc_stderr": 0.023888375986348194
},
"analogies_sk_SK_AVERAGE|0": {
"acc": 0.000628693574751666,
"acc_stderr": 0.00039549148554889166,
"acc__prompts": {
"3": 0.0,
"2": 0.0007544322897019992,
"1": 0.001131648434552999
}
},
"analogies_sk_AVERAGE|0": {
"acc": 0.000628693574751666,
"acc_stderr": 0.00039549148554889166
},
"SK_AVERAGE|0": {
"em_norm_relaxed": 0.20551192367489274,
"acc": 0.21129673365203147,
"em_relaxed": 0.24052159099864373,
"rouge2": 0.00315582529372049,
"rouge1": 0.014859863107823173,
"rougeL": 0.014859863107823173,
"qf1": 0.07079407895918113,
"qem": 0.038181737423518054,
"prompt_level_strict_acc": 0.10242085661080075,
"prompt_level_loose_acc": 0.10242085661080075,
"inst_level_strict_acc": 0.21307506053268765,
"inst_level_loose_acc": 0.21307506053268765,
"em_norm_relaxed_stderr": 0.0025722619842950684,
"acc_stderr": 0.0031822329617778393,
"em_relaxed_stderr": 0.0056971140284625835,
"rouge2_stderr": 0.001710070063230727,
"rouge1_stderr": 0.0047917616644827035,
"rougeL_stderr": 0.0047917616644827035,
"qf1_stderr": 0.002234105218066023,
"qem_stderr": 0.0029234626947495608,
"prompt_level_strict_acc_stderr": 0.01309628698247887,
"prompt_level_loose_acc_stderr": 0.01309628698247887,
"inst_level_strict_acc_stderr": 0.00041218013349389904,
"inst_level_loose_acc_stderr": 0.00041218013349389904
},
"AVERAGE|0": {
"em_norm_relaxed": 0.185433666807816,
"acc": 0.21317053562216026,
"em_relaxed": 0.2602872791374196,
"rouge2": 0.00315582529372049,
"rouge1": 0.014859863107823173,
"rougeL": 0.014859863107823173,
"qf1": 0.07079407895918113,
"qem": 0.038181737423518054,
"prompt_level_strict_acc": 0.10242085661080075,
"prompt_level_loose_acc": 0.10242085661080075,
"inst_level_strict_acc": 0.21307506053268765,
"inst_level_loose_acc": 0.21307506053268765,
"em_norm_relaxed_stderr": 0.002756064620435493,
"acc_stderr": 0.003457735075973522,
"em_relaxed_stderr": 0.0061656193838727944,
"rouge2_stderr": 0.001710070063230727,
"rouge1_stderr": 0.0047917616644827035,
"rougeL_stderr": 0.0047917616644827035,
"qf1_stderr": 0.002234105218066023,
"qem_stderr": 0.0029234626947495608,
"prompt_level_strict_acc_stderr": 0.01309628698247887,
"prompt_level_loose_acc_stderr": 0.01309628698247887,
"inst_level_strict_acc_stderr": 0.00041218013349389904,
"inst_level_loose_acc_stderr": 0.00041218013349389904
}
}
}