{ "model_name": "models/ministral-last-109064", "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "max_samples": null, "job_id": "0", "start_time": 883355.295419078, "end_time": 883503.394399547, "total_evaluation_time_secondes": "148.0989804690471", "model_config": { "model_name": "models/ministral-last-109064", "generation_parameters": { "num_blocks": null, "block_size": null, "early_stopping": null, "repetition_penalty": null, "frequency_penalty": null, "length_penalty": null, "presence_penalty": null, "max_new_tokens": null, "min_new_tokens": null, "seed": null, "stop_tokens": null, "temperature": 0, "top_k": null, "min_p": null, "top_p": null, "truncate_prompt": null, "cache_implementation": null, "response_format": null, "skip_special_tokens": true }, "system_prompt": null, "cache_dir": "~/.cache/huggingface/lighteval", "tokenizer": null, "revision": "main", "dtype": "bfloat16", "tensor_parallel_size": 1, "data_parallel_size": 1, "pipeline_parallel_size": 1, "gpu_memory_utilization": 0.9, "enable_prefix_caching": null, "max_model_length": null, "quantization": null, "load_format": null, "swap_space": 4, "seed": 1234, "trust_remote_code": true, "add_special_tokens": true, "multichoice_continuations_start_space": true, "pairwise_tokenization": false, "max_num_seqs": 128, "max_num_batched_tokens": 2048, "subfolder": null, "is_async": false, "override_chat_template": null, "enforce_eager": false }, "model_name": "models/ministral-last-109064" }, "versions": {}, "results": { "movie_review.prompt1_sentiment_binary_sk|0": { "acc": 0.09860557768924302, "acc_stderr": 0.006654792030522998, "em_norm_relaxed": 0.10109561752988047, "em_norm_relaxed_stderr": 0.006728979843112432 }, "e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": { "acc": 0.1, "acc_stderr": 0.06882472016116853 }, "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { "acc": 0.21983914209115282, "acc_stderr": 0.021472030675572816, "em_relaxed": 0.21983914209115282, "em_relaxed_stderr": 0.021472030675572816 }, "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { "acc": 0.27350427350427353, "acc_stderr": 0.029202540153431183, "em_relaxed": 0.27350427350427353, "em_relaxed_stderr": 0.029202540153431183 }, "belebele.prompt3_mqa_single_correct_sk|0": { "acc": 0.27111111111111114, "acc_stderr": 0.014826016446581967, "em_relaxed": 0.2722222222222222, "em_relaxed_stderr": 0.014845038794433947 }, "e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": { "acc": 0.2318840579710145, "acc_stderr": 0.05117930441535768, "em_relaxed": 0.2318840579710145, "em_relaxed_stderr": 0.05117930441535768 }, "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { "acc": 0.254601226993865, "acc_stderr": 0.024164783161712056, "em_relaxed": 0.254601226993865, "em_relaxed_stderr": 0.024164783161712056 }, "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { "acc": 0.20202020202020202, "acc_stderr": 0.0405583530232473, "em_relaxed": 0.2222222222222222, "em_relaxed_stderr": 0.041996052556580815 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { "acc": 0.20522388059701493, "acc_stderr": 0.017460611985170203, "em_relaxed": 0.20522388059701493, "em_relaxed_stderr": 0.017460611985170203 }, "senti_sk.prompt2_sentiment_binary_sk|0": { "acc": 0.32135193133047213, "acc_stderr": 0.007649500362664599, "em_norm_relaxed": 0.3261802575107296, "em_norm_relaxed_stderr": 0.007679288875174346 }, "slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": { "acc": 0.18095238095238095, "acc_stderr": 0.03775026958386237, "em_relaxed": 0.18095238095238095, "em_relaxed_stderr": 0.03775026958386237 }, "e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": { "acc": 0.05357142857142857, "acc_stderr": 0.030361917118846844 }, "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { "acc": 0.27181208053691275, "acc_stderr": 0.018238857466939602, "em_relaxed": 0.27181208053691275, "em_relaxed_stderr": 0.018238857466939602 }, "e_tests.prompt3_qa_sk:chemia|0": { "qem": 0.037037037037037035, "qem_stderr": 0.025940896147128693, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.02444444444444445, "rouge1_stderr": 0.011223747742795042, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.02444444444444445, "rougeL_stderr": 0.011223747742795042 }, "scio.prompt3_mqa_single_correct_sk:vsp|0": { "acc": 0.2657766990291262, "acc_stderr": 0.015398294286338463, "em_relaxed": 0.27184466019417475, "em_relaxed_stderr": 0.015508596797488939 }, "arc.prompt2_mqa_single_correct_sk|0": { "acc": 0.24123182207014543, "acc_stderr": 0.01251844632394384, "em_relaxed": 0.24721984602224123, "em_relaxed_stderr": 0.012622759999132595 }, "sklep.prompt3_nli_binary_sk:rte|0": { "acc": 0.1650602409638554, "acc_stderr": 0.009114348186638069, "em_norm_relaxed": 0.16566265060240964, "em_norm_relaxed_stderr": 0.009127670467992892 }, "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { "acc": 0.1513157894736842, "acc_stderr": 0.029162631596843982, "em_relaxed": 0.1513157894736842, "em_relaxed_stderr": 0.029162631596843982 }, "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { "acc": 0.25892857142857145, "acc_stderr": 0.041577515398656284, "em_relaxed": 0.2767857142857143, "em_relaxed_stderr": 0.04246624336697624 }, "driving_test.prompt3_mqa_single_correct_sk|0": { "acc": 0.17261055634807418, "acc_stderr": 0.014283664605079543, "em_relaxed": 0.17261055634807418, "em_relaxed_stderr": 0.014283664605079543 }, "e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { "acc": 0.23232323232323232, "acc_stderr": 0.04266016017054689, "em_relaxed": 0.23232323232323232, "em_relaxed_stderr": 0.04266016017054689 }, "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { "acc": 0.24679760888129804, "acc_stderr": 0.01260472537669998, "em_relaxed": 0.2493595217762596, "em_relaxed_stderr": 0.012648412983648835 }, "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { "acc": 0.25217391304347825, "acc_stderr": 0.040672227541547167, "em_relaxed": 0.2608695652173913, "em_relaxed_stderr": 0.04112631751856164 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { "acc": 0.2676767676767677, "acc_stderr": 0.03154449888270286, "em_relaxed": 0.2676767676767677, "em_relaxed_stderr": 0.03154449888270286 }, "senti_sk.prompt1_sentiment_binary_sk|0": { "acc": 0.39914163090128757, "acc_stderr": 0.0080217638856134, "em_norm_relaxed": 0.4069206008583691, "em_norm_relaxed_stderr": 0.008046954795106477 }, "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { "acc": 0.1833013435700576, "acc_stderr": 0.011991910538467281, "em_norm_relaxed": 0.19193857965451055, "em_norm_relaxed_stderr": 0.012206129123492155 }, "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { "acc": 0.2222222222222222, "acc_stderr": 0.03188578017686398, "em_relaxed": 0.2222222222222222, "em_relaxed_stderr": 0.03188578017686398 }, "e_tests.prompt2_qa_sk:fyzika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.014285714285714287, "rouge1_stderr": 0.014285714285714287, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.014285714285714287, "rougeL_stderr": 0.014285714285714287 }, "scio.prompt2_mqa_single_correct_sk:che|0": { "acc": 0.2601626016260163, "acc_stderr": 0.028028995361669362, "em_relaxed": 0.2601626016260163, "em_relaxed_stderr": 0.028028995361669362 }, "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { "acc": 0.17, "acc_stderr": 0.0377525168068637, "em_relaxed": 0.17, "em_relaxed_stderr": 0.0377525168068637 }, "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { "acc": 0.2408198121263877, "acc_stderr": 0.012500448888138267, "em_relaxed": 0.2408198121263877, "em_relaxed_stderr": 0.012500448888138267 }, "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { "acc": 0.25153374233128833, "acc_stderr": 0.024068141649530538, "em_relaxed": 0.25153374233128833, "em_relaxed_stderr": 0.024068141649530538 }, "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { "acc": 0.22826086956521738, "acc_stderr": 0.025309563050037266, "em_relaxed": 0.22826086956521738, "em_relaxed_stderr": 0.025309563050037266 }, "e_tests.prompt2_qa_sk:geografia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.01904761904761905, "qf1_stderr": 0.019047619047619056, "rouge1": 0.01360544217687075, "rouge1_stderr": 0.013605442176870748, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.01360544217687075, "rougeL_stderr": 0.013605442176870748 }, "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { "acc": 0.29292929292929293, "acc_stderr": 0.045972676254181784, "em_relaxed": 0.29292929292929293, "em_relaxed_stderr": 0.045972676254181784 }, "e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": { "acc": 0.05, "acc_stderr": 0.04999999999999998 }, "e_tests.prompt2_mqa_single_correct_sk:fyzika|0": { "acc": 0.13636363636363635, "acc_stderr": 0.07488677009526491, "em_relaxed": 0.13636363636363635, "em_relaxed_stderr": 0.07488677009526491 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { "acc": 0.24067164179104478, "acc_stderr": 0.018482060138670608, "em_relaxed": 0.24253731343283583, "em_relaxed_stderr": 0.01853075065058218 }, "slovakcopa.prompt1_copa_sk|0": { "acc": 0.402, "acc_stderr": 0.021948929609938612, "em_relaxed": 0.402, "em_relaxed_stderr": 0.021948929609938612 }, "movie_review.prompt3_sentiment_binary_sk|0": { "acc": 0.20766932270916336, "acc_stderr": 0.009054526051113992, "em_norm_relaxed": 0.20916334661354583, "em_norm_relaxed_stderr": 0.009078466587423147 }, "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { "acc": 0.20348837209302326, "acc_stderr": 0.03078703062133896, "em_relaxed": 0.20348837209302326, "em_relaxed_stderr": 0.03078703062133896 }, "e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": { "acc": 0.034482758620689655, "acc_stderr": 0.034482758620689676 }, "e_tests.prompt3_mqa_single_correct_sk:geografia|0": { "acc": 0.20408163265306123, "acc_stderr": 0.05817221556628253, "em_relaxed": 0.20408163265306123, "em_relaxed_stderr": 0.05817221556628253 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { "acc": 0.22932330827067668, "acc_stderr": 0.02582481829225878, "em_relaxed": 0.22932330827067668, "em_relaxed_stderr": 0.02582481829225878 }, "sklep.prompt2_qa_sk:question-answering|0": { "qem": 0.05238443076280914, "qem_stderr": 0.0022760883977866997, "qf1": 0.14428713100967, "qf1_stderr": 0.002729924096699559 }, "e_tests.prompt2_qa_sk:informatika|0": { "qem": 0.038461538461538464, "qem_stderr": 0.021915495473007307, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0036324786324786326, "rouge1_stderr": 0.0025563439430967027, "rouge2": 0.0008547008547008547, "rouge2_stderr": 0.0008547008547008546, "rougeL": 0.0036324786324786326, "rougeL_stderr": 0.0025563439430967027 }, "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { "acc": 0.32340425531914896, "acc_stderr": 0.030579442773610337, "em_relaxed": 0.32340425531914896, "em_relaxed_stderr": 0.030579442773610337 }, "e_tests.prompt1_qa_sk:dejepis|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { "acc": 0.2482758620689655, "acc_stderr": 0.03600105692727771, "em_relaxed": 0.2482758620689655, "em_relaxed_stderr": 0.03600105692727771 }, "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { "acc": 0.3926940639269406, "acc_stderr": 0.03307521564995785, "em_relaxed": 0.3926940639269406, "em_relaxed_stderr": 0.03307521564995785 }, "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { "acc": 0.31, "acc_stderr": 0.04648231987117316, "em_relaxed": 0.31, "em_relaxed_stderr": 0.04648231987117316 }, "e_tests.prompt1_qa_sk:obcianska-nauka|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.020408163265306124, "rouge1_stderr": 0.020408163265306124, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.020408163265306124, "rougeL_stderr": 0.020408163265306124 }, "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { "acc": 0.2548262548262548, "acc_stderr": 0.027129450992473495, "em_relaxed": 0.2625482625482625, "em_relaxed_stderr": 0.027394382974098772 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { "acc": 0.19626168224299065, "acc_stderr": 0.027213555922450262, "em_relaxed": 0.205607476635514, "em_relaxed_stderr": 0.027691547344010747 }, "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { "acc": 0.23478260869565218, "acc_stderr": 0.039698395317531235, "em_relaxed": 0.23478260869565218, "em_relaxed_stderr": 0.039698395317531235 }, "senti_sk.prompt3_sentiment_3_class_sk|0": { "acc": 0.573444515715202, "acc_stderr": 0.007232629453324713, "em_norm_relaxed": 0.5809279452640581, "em_norm_relaxed_stderr": 0.007215529950821413 }, "e_tests.prompt1_qa_sk:biologia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.019047619047619046, "qf1_stderr": 0.019047619047619035, "rouge1": 0.04, "rouge1_stderr": 0.022529159443043678, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.04, "rougeL_stderr": 0.022529159443043678 }, "slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": { "acc": 0.27371273712737126, "acc_stderr": 0.023242230741504617, "em_relaxed": 0.27371273712737126, "em_relaxed_stderr": 0.023242230741504617 }, "mmlu.prompt1_mqa_single_correct_sk:virology|0": { "acc": 0.35365853658536583, "acc_stderr": 0.03744805613781578, "em_relaxed": 0.35365853658536583, "em_relaxed_stderr": 0.03744805613781578 }, "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { "acc": 0.23857868020304568, "acc_stderr": 0.03044390069569734, "em_relaxed": 0.23857868020304568, "em_relaxed_stderr": 0.03044390069569734 }, "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { "acc": 0.3926940639269406, "acc_stderr": 0.03307521564995785, "em_relaxed": 0.3926940639269406, "em_relaxed_stderr": 0.03307521564995785 }, "e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": { "acc": 0.022727272727272728, "acc_stderr": 0.022727272727272728 }, "mmlu.prompt2_mqa_single_correct_sk:virology|0": { "acc": 0.2926829268292683, "acc_stderr": 0.035637888362588285, "em_relaxed": 0.2926829268292683, "em_relaxed_stderr": 0.035637888362588285 }, "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { "acc": 0.2323943661971831, "acc_stderr": 0.03556905428023102, "em_relaxed": 0.23943661971830985, "em_relaxed_stderr": 0.03593796120702438 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { "acc": 0.2534562211981567, "acc_stderr": 0.02959731008909016, "em_relaxed": 0.2534562211981567, "em_relaxed_stderr": 0.02959731008909016 }, "multi_wiki_qa.prompt3_qa_sk|0": { "qem": 0.07884730838503101, "qem_stderr": 0.0038128351969214953, "qf1": 0.12171966052606027, "qf1_stderr": 0.003955037338259647 }, "e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": { "acc": 0.02857142857142857, "acc_stderr": 0.014130710990583526 }, "e_tests.prompt2_mqa_single_correct_sk:biologia|0": { "acc": 0.3103448275862069, "acc_stderr": 0.049887188500387446, "em_relaxed": 0.3103448275862069, "em_relaxed_stderr": 0.049887188500387446 }, "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { "acc": 0.24, "acc_stderr": 0.04292346959909284, "em_relaxed": 0.24, "em_relaxed_stderr": 0.04292346959909284 }, "e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": { "acc": 0.034482758620689655, "acc_stderr": 0.034482758620689676 }, "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { "acc": 0.29, "acc_stderr": 0.045604802157206845, "em_relaxed": 0.29, "em_relaxed_stderr": 0.045604802157206845 }, "ifeval_sk.ifeval|0": { "inst_level_loose_acc": 0.21307506053268765, "inst_level_loose_acc_stderr": 0.00041218013349389904, "inst_level_strict_acc": 0.21307506053268765, "inst_level_strict_acc_stderr": 0.00041218013349389904, "prompt_level_loose_acc": 0.10242085661080075, "prompt_level_loose_acc_stderr": 0.01309628698247887, "prompt_level_strict_acc": 0.10242085661080075, "prompt_level_strict_acc_stderr": 0.01309628698247887 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { "acc": 0.18181818181818182, "acc_stderr": 0.027479603010538804, "em_relaxed": 0.18181818181818182, "em_relaxed_stderr": 0.027479603010538804 }, "e_tests.prompt1_qa_sk:geografia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { "acc": 0.22429906542056074, "acc_stderr": 0.015251413306651283, "em_relaxed": 0.22429906542056074, "em_relaxed_stderr": 0.015251413306651283 }, "e_tests.prompt3_qa_sk:slovensky-jazyk|0": { "qem": 0.0051813471502590676, "qem_stderr": 0.00518134715025908, "qf1": 0.008737173625927054, "qf1_stderr": 0.005740091786106941, "rouge1": 0.016785287251608494, "rouge1_stderr": 0.006781696658405232, "rouge2": 0.00659444182760245, "rouge2_stderr": 0.005363480846922639, "rougeL": 0.016785287251608494, "rougeL_stderr": 0.006781696658405232 }, "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { "acc": 0.26495726495726496, "acc_stderr": 0.028911208802749472, "em_relaxed": 0.26495726495726496, "em_relaxed_stderr": 0.028911208802749472 }, "driving_test.prompt1_mqa_single_correct_sk|0": { "acc": 0.3666191155492154, "acc_stderr": 0.01821339929295096, "em_relaxed": 0.3666191155492154, "em_relaxed_stderr": 0.01821339929295096 }, "scio.prompt1_mqa_single_correct_sk:aj|0": { "acc": 0.2, "acc_stderr": 0.13333333333333333, "em_relaxed": 0.2, "em_relaxed_stderr": 0.13333333333333333 }, "sklep.prompt2_nli_binary_sk:rte|0": { "acc": 0.5132530120481927, "acc_stderr": 0.012271402375351256, "em_norm_relaxed": 0.5132530120481927, "em_norm_relaxed_stderr": 0.012271402375351256 }, "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { "acc": 0.23857868020304568, "acc_stderr": 0.03044390069569734, "em_relaxed": 0.23857868020304568, "em_relaxed_stderr": 0.03044390069569734 }, "multi_wiki_qa.prompt2_qa_sk|0": { "qem": 0.08204922953772263, "qem_stderr": 0.003882716972284686, "qf1": 0.12697709233547141, "qf1_stderr": 0.0040052643032774445 }, "e_tests.prompt2_mqa_single_correct_sk:geografia|0": { "acc": 0.22448979591836735, "acc_stderr": 0.06022425581505364, "em_relaxed": 0.22448979591836735, "em_relaxed_stderr": 0.06022425581505364 }, "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { "acc": 0.26, "acc_stderr": 0.0440844002276808, "em_relaxed": 0.26, "em_relaxed_stderr": 0.0440844002276808 }, "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { "acc": 0.22442244224422442, "acc_stderr": 0.024007242951044422, "em_relaxed": 0.2376237623762376, "em_relaxed_stderr": 0.0244921064409582 }, "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { "acc": 0.2, "acc_stderr": 0.04020151261036845, "em_relaxed": 0.2, "em_relaxed_stderr": 0.04020151261036845 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { "acc": 0.21705426356589147, "acc_stderr": 0.020982456910832083, "em_relaxed": 0.21705426356589147, "em_relaxed_stderr": 0.020982456910832083 }, "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { "acc": 0.2875, "acc_stderr": 0.022658174179374095, "em_relaxed": 0.2925, "em_relaxed_stderr": 0.022774020616430368 }, "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { "acc": 0.23166023166023167, "acc_stderr": 0.02626591571601146, "em_relaxed": 0.23166023166023167, "em_relaxed_stderr": 0.02626591571601146 }, "e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": { "acc": 0.034482758620689655, "acc_stderr": 0.034482758620689676 }, "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { "acc": 0.2413793103448276, "acc_stderr": 0.02399654973117778, "em_relaxed": 0.2413793103448276, "em_relaxed_stderr": 0.02399654973117778 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { "acc": 0.18666666666666668, "acc_stderr": 0.0319208566416966, "em_relaxed": 0.18666666666666668, "em_relaxed_stderr": 0.0319208566416966 }, "sklep.prompt3_nli_3_classes_sk:nli|0": { "acc": 0.002398081534772182, "acc_stderr": 0.0006915050545011868, "em_norm_relaxed": 0.002398081534772182, "em_norm_relaxed_stderr": 0.0006915050545011868 }, "e_tests.prompt3_qa_sk:fyzika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.014285714285714287, "rouge1_stderr": 0.014285714285714287, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.014285714285714287, "rougeL_stderr": 0.014285714285714287 }, "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { "acc": 0.25892857142857145, "acc_stderr": 0.041577515398656284, "em_relaxed": 0.25892857142857145, "em_relaxed_stderr": 0.041577515398656284 }, "mmlu.prompt3_mqa_single_correct_sk:virology|0": { "acc": 0.3048780487804878, "acc_stderr": 0.03605784583600454, "em_relaxed": 0.3048780487804878, "em_relaxed_stderr": 0.03605784583600454 }, "e_tests.prompt3_mqa_single_correct_sk:informatika|0": { "acc": 0.224, "acc_stderr": 0.03744070581082714, "em_relaxed": 0.224, "em_relaxed_stderr": 0.03744070581082714 }, "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { "acc": 0.28498074454428757, "acc_stderr": 0.016183663085909714, "em_relaxed": 0.28498074454428757, "em_relaxed_stderr": 0.016183663085909714 }, "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { "acc": 0.25838926174496646, "acc_stderr": 0.01794596235855172, "em_relaxed": 0.26174496644295303, "em_relaxed_stderr": 0.01802120781376133 }, "arc.prompt3_mqa_single_correct_sk|0": { "acc": 0.24550898203592814, "acc_stderr": 0.012593293013246194, "em_relaxed": 0.24550898203592814, "em_relaxed_stderr": 0.012593293013246194 }, "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { "acc": 0.3191489361702128, "acc_stderr": 0.030472973363380035, "em_relaxed": 0.3191489361702128, "em_relaxed_stderr": 0.030472973363380035 }, "scio.prompt1_mqa_single_correct_sk:vsp|0": { "acc": 0.24514563106796117, "acc_stderr": 0.014994906005686719, "em_relaxed": 0.24514563106796117, "em_relaxed_stderr": 0.014994906005686719 }, "e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": { "acc": 0.03571428571428571, "acc_stderr": 0.025023180348048964 }, "e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": { "acc": 0.27074235807860264, "acc_stderr": 0.02078550104939707, "em_relaxed": 0.27074235807860264, "em_relaxed_stderr": 0.02078550104939707 }, "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { "acc": 0.2631578947368421, "acc_stderr": 0.03377310252209194, "em_relaxed": 0.2631578947368421, "em_relaxed_stderr": 0.03377310252209194 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { "acc": 0.26262626262626265, "acc_stderr": 0.04445287676983945, "em_relaxed": 0.26262626262626265, "em_relaxed_stderr": 0.04445287676983945 }, "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { "acc": 0.26785714285714285, "acc_stderr": 0.04203277291467762, "em_relaxed": 0.2857142857142857, "em_relaxed_stderr": 0.04287858751340456 }, "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { "acc": 0.16216216216216217, "acc_stderr": 0.02479468591119148, "em_relaxed": 0.16666666666666666, "em_relaxed_stderr": 0.02506903489984114 }, "belebele.prompt1_mqa_single_correct_sk|0": { "acc": 0.2577777777777778, "acc_stderr": 0.014588474089651667, "em_relaxed": 0.2577777777777778, "em_relaxed_stderr": 0.014588474089651667 }, "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { "acc": 0.26785714285714285, "acc_stderr": 0.04203277291467762, "em_relaxed": 0.2767857142857143, "em_relaxed_stderr": 0.042466243366976256 }, "slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.024691358024691357, "qf1_stderr": 0.024691358024691353, "rouge1": 0.01851851851851852, "rouge1_stderr": 0.01851851851851852, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.01851851851851852, "rougeL_stderr": 0.01851851851851852 }, "movie_review.prompt2_sentiment_binary_sk|0": { "acc": 0.13097609561752988, "acc_stderr": 0.007530749388005803, "em_norm_relaxed": 0.13645418326693226, "em_norm_relaxed_stderr": 0.007662357895460915 }, "e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": { "acc": 0.13043478260869565, "acc_stderr": 0.04084071463598805, "em_relaxed": 0.14492753623188406, "em_relaxed_stderr": 0.042689636687312106 }, "e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { "acc": 0.20930232558139536, "acc_stderr": 0.03110958390976464, "em_relaxed": 0.20930232558139536, "em_relaxed_stderr": 0.03110958390976464 }, "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { "acc": 0.2277992277992278, "acc_stderr": 0.026111473881583542, "em_relaxed": 0.2277992277992278, "em_relaxed_stderr": 0.026111473881583542 }, "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { "acc": 0.2734274711168164, "acc_stderr": 0.015979778720725094, "em_relaxed": 0.2734274711168164, "em_relaxed_stderr": 0.015979778720725094 }, "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { "acc": 0.22784810126582278, "acc_stderr": 0.033475282640609255, "em_relaxed": 0.22784810126582278, "em_relaxed_stderr": 0.033475282640609255 }, "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { "acc": 0.29906542056074764, "acc_stderr": 0.044470182376718334, "em_relaxed": 0.29906542056074764, "em_relaxed_stderr": 0.044470182376718334 }, "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { "acc": 0.21568627450980393, "acc_stderr": 0.04092563958237656, "em_relaxed": 0.21568627450980393, "em_relaxed_stderr": 0.04092563958237656 }, "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { "acc": 0.2064343163538874, "acc_stderr": 0.020985093876713833, "em_relaxed": 0.2064343163538874, "em_relaxed_stderr": 0.020985093876713833 }, "slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": { "acc": 0.25203252032520324, "acc_stderr": 0.022633188466788714, "em_relaxed": 0.25203252032520324, "em_relaxed_stderr": 0.022633188466788714 }, "e_tests.prompt3_qa_sk:dejepis|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "scio.prompt1_mqa_single_correct_sk:mat|0": { "acc": 0.1885245901639344, "acc_stderr": 0.020472706124868213, "em_relaxed": 0.1912568306010929, "em_relaxed_stderr": 0.020585781457546286 }, "e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": { "acc": 0.02857142857142857, "acc_stderr": 0.028571428571428574 }, "scio.prompt3_mqa_single_correct_sk:mat|0": { "acc": 0.31420765027322406, "acc_stderr": 0.024297313836707593, "em_relaxed": 0.31693989071038253, "em_relaxed_stderr": 0.024354066150157113 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { "acc": 0.22340425531914893, "acc_stderr": 0.024847921358063962, "em_relaxed": 0.23404255319148937, "em_relaxed_stderr": 0.025257861359432414 }, "slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": { "acc": 0.14285714285714285, "acc_stderr": 0.03431317581537582, "em_relaxed": 0.14285714285714285, "em_relaxed_stderr": 0.03431317581537582 }, "slovakcopa.prompt2_copa_sk|0": { "acc": 0.392, "acc_stderr": 0.021854684955611256, "em_relaxed": 0.394, "em_relaxed_stderr": 0.021874299301689253 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { "acc": 0.22962962962962963, "acc_stderr": 0.036333844140734636, "em_relaxed": 0.22962962962962963, "em_relaxed_stderr": 0.036333844140734636 }, "e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": { "acc": 0.02857142857142857, "acc_stderr": 0.014130710990583526 }, "belebele.prompt2_mqa_single_correct_sk|0": { "acc": 0.27111111111111114, "acc_stderr": 0.014826016446581962, "em_relaxed": 0.27111111111111114, "em_relaxed_stderr": 0.014826016446581962 }, "skj1.prompt1_skj1_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_relaxed": 0.01, "em_relaxed_stderr": 0.009999999999999997 }, "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { "acc": 0.26, "acc_stderr": 0.0440844002276808, "em_relaxed": 0.26, "em_relaxed_stderr": 0.0440844002276808 }, "driving_test.prompt2_mqa_single_correct_sk|0": { "acc": 0.2738944365192582, "acc_stderr": 0.01685553212123784, "em_relaxed": 0.2738944365192582, "em_relaxed_stderr": 0.01685553212123784 }, "scio.prompt3_mqa_single_correct_sk:aj|0": { "acc": 0.3, "acc_stderr": 0.15275252316519464, "em_relaxed": 0.3, "em_relaxed_stderr": 0.15275252316519464 }, "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { "acc": 0.26, "acc_stderr": 0.04408440022768078, "em_relaxed": 0.26, "em_relaxed_stderr": 0.04408440022768078 }, "scio.prompt1_mqa_single_correct_sk:bio|0": { "acc": 0.21238938053097345, "acc_stderr": 0.022246609446382862, "em_relaxed": 0.21238938053097345, "em_relaxed_stderr": 0.022246609446382862 }, "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { "acc": 0.24, "acc_stderr": 0.04292346959909283, "em_relaxed": 0.24, "em_relaxed_stderr": 0.04292346959909283 }, "skj1.prompt3_skj1_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_relaxed": 0.0, "em_relaxed_stderr": 0.0 }, "skj1.prompt2_skj1_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_relaxed": 0.0, "em_relaxed_stderr": 0.0 }, "e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": { "acc": 0.02857142857142857, "acc_stderr": 0.028571428571428574 }, "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { "acc": 0.22033898305084745, "acc_stderr": 0.03831824849223319, "em_relaxed": 0.22033898305084745, "em_relaxed_stderr": 0.03831824849223319 }, "e_tests.prompt1_mqa_single_correct_sk:dejepis|0": { "acc": 0.1875, "acc_stderr": 0.05693291124629694, "em_relaxed": 0.1875, "em_relaxed_stderr": 0.05693291124629694 }, "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { "acc": 0.23391812865497075, "acc_stderr": 0.03246721765117826, "em_relaxed": 0.23391812865497075, "em_relaxed_stderr": 0.03246721765117826 }, "slovak_tests.prompt1_qa_sk:maturita|0": { "qem": 0.008849557522123894, "qem_stderr": 0.0062436609110668876, "qf1": 0.007079646017699116, "qf1_stderr": 0.0038256704785137783, "rouge1": 0.014790713408591503, "rouge1_stderr": 0.004864012429457399, "rouge2": 0.003448518050287962, "rouge2_stderr": 0.002428770392002376, "rougeL": 0.014790713408591503, "rougeL_stderr": 0.004864012429457399 }, "sklep.prompt1_qa_sk:question-answering|0": { "qem": 0.05248878221851195, "qem_stderr": 0.002278228840555749, "qf1": 0.1424834912487429, "qf1_stderr": 0.0027244148341408567 }, "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { "acc": 0.2725, "acc_stderr": 0.022290166077429768, "em_relaxed": 0.2725, "em_relaxed_stderr": 0.022290166077429768 }, "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { "acc": 0.24846625766871167, "acc_stderr": 0.02396990269668754, "em_relaxed": 0.24846625766871167, "em_relaxed_stderr": 0.02396990269668754 }, "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { "acc": 0.18468468468468469, "acc_stderr": 0.026102500004017648, "em_relaxed": 0.18468468468468469, "em_relaxed_stderr": 0.026102500004017648 }, "e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": { "acc": 0.03571428571428571, "acc_stderr": 0.03571428571428572 }, "sklep.prompt1_nli_3_classes_sk:nli|0": { "acc": 0.004396482813749001, "acc_stderr": 0.0009353639970831211, "em_norm_relaxed": 0.004996003197442046, "em_norm_relaxed_stderr": 0.0009968011147772275 }, "e_tests.prompt2_qa_sk:obcianska-nauka|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.020408163265306124, "rouge1_stderr": 0.020408163265306124, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.020408163265306124, "rougeL_stderr": 0.020408163265306124 }, "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { "acc": 0.24358974358974358, "acc_stderr": 0.028120966503914394, "em_relaxed": 0.24358974358974358, "em_relaxed_stderr": 0.028120966503914394 }, "financial_exams.prompt3_completion_sk|0": { "acc": 0.18590704647676162, "acc_stderr": 0.010655402235931219, "em_relaxed": 0.18590704647676162, "em_relaxed_stderr": 0.010655402235931219 }, "arc.prompt1_mqa_single_correct_sk|0": { "acc": 0.24721984602224123, "acc_stderr": 0.012622759999132588, "em_relaxed": 0.24721984602224123, "em_relaxed_stderr": 0.012622759999132588 }, "slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": { "acc": 0.30414746543778803, "acc_stderr": 0.0313021257639245, "em_relaxed": 0.30414746543778803, "em_relaxed_stderr": 0.0313021257639245 }, "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { "acc": 0.09213051823416507, "acc_stderr": 0.00896371626415267, "em_norm_relaxed": 0.09404990403071017, "em_norm_relaxed_stderr": 0.009047028350861394 }, "e_tests.prompt2_mqa_single_correct_sk:matematika|0": { "acc": 0.15384615384615385, "acc_stderr": 0.05852968540405248, "em_relaxed": 0.15384615384615385, "em_relaxed_stderr": 0.05852968540405248 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { "acc": 0.20567375886524822, "acc_stderr": 0.024112138950471887, "em_relaxed": 0.20567375886524822, "em_relaxed_stderr": 0.024112138950471887 }, "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { "acc": 0.2601880877742947, "acc_stderr": 0.024603155110002158, "em_relaxed": 0.26332288401253917, "em_relaxed_stderr": 0.02469842907233297 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { "acc": 0.1822429906542056, "acc_stderr": 0.026451344562470145, "em_relaxed": 0.1822429906542056, "em_relaxed_stderr": 0.026451344562470145 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { "acc": 0.22807017543859648, "acc_stderr": 0.056069817847611766, "em_relaxed": 0.22807017543859648, "em_relaxed_stderr": 0.056069817847611766 }, "e_tests.prompt1_qa_sk:fyzika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.014285714285714287, "rouge1_stderr": 0.014285714285714287, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.014285714285714287, "rougeL_stderr": 0.014285714285714287 }, "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { "acc": 0.24242424242424243, "acc_stderr": 0.04329004329004326, "em_relaxed": 0.24242424242424243, "em_relaxed_stderr": 0.04329004329004326 }, "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { "acc": 0.19, "acc_stderr": 0.03942772444036623, "em_relaxed": 0.19, "em_relaxed_stderr": 0.03942772444036623 }, "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { "acc": 0.2675, "acc_stderr": 0.022160484939004153, "em_relaxed": 0.27, "em_relaxed_stderr": 0.022225772481648733 }, "e_tests.prompt2_qa_sk:slovensky-jazyk|0": { "qem": 0.0051813471502590676, "qem_stderr": 0.00518134715025908, "qf1": 0.010905502097211939, "qf1_stderr": 0.0062654994299230354, "rouge1": 0.015304902351534477, "rouge1_stderr": 0.006747971135704163, "rouge2": 0.0072538860103626935, "rouge2_stderr": 0.005570450232424934, "rougeL": 0.015304902351534477, "rougeL_stderr": 0.006747971135704163 }, "slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": { "acc": 0.30414746543778803, "acc_stderr": 0.0313021257639245, "em_relaxed": 0.30414746543778803, "em_relaxed_stderr": 0.0313021257639245 }, "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { "acc": 0.3515981735159817, "acc_stderr": 0.03233829906905826, "em_relaxed": 0.3515981735159817, "em_relaxed_stderr": 0.03233829906905826 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { "acc": 0.2706766917293233, "acc_stderr": 0.027293707912941223, "em_relaxed": 0.2706766917293233, "em_relaxed_stderr": 0.027293707912941223 }, "e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { "acc": 0.27522935779816515, "acc_stderr": 0.042976970405367555, "em_relaxed": 0.28440366972477066, "em_relaxed_stderr": 0.043409998674551635 }, "slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": { "acc": 0.2791327913279133, "acc_stderr": 0.02338348056654487, "em_relaxed": 0.2791327913279133, "em_relaxed_stderr": 0.02338348056654487 }, "slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": { "acc": 0.26666666666666666, "acc_stderr": 0.04336290903919941, "em_relaxed": 0.26666666666666666, "em_relaxed_stderr": 0.04336290903919941 }, "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { "acc": 0.1513157894736842, "acc_stderr": 0.029162631596843982, "em_relaxed": 0.1513157894736842, "em_relaxed_stderr": 0.029162631596843982 }, "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { "acc": 0.2828282828282828, "acc_stderr": 0.04549461117008498, "em_relaxed": 0.2828282828282828, "em_relaxed_stderr": 0.04549461117008498 }, "e_tests.prompt3_mqa_single_correct_sk:biologia|0": { "acc": 0.22988505747126436, "acc_stderr": 0.045371581852507746, "em_relaxed": 0.22988505747126436, "em_relaxed_stderr": 0.045371581852507746 }, "e_tests.prompt1_qa_sk:chemia|0": { "qem": 0.018518518518518517, "qem_stderr": 0.018518518518518514, "qf1": 0.012345679012345678, "qf1_stderr": 0.012345679012345675, "rouge1": 0.02580246913580247, "rouge1_stderr": 0.011283149325161083, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.02580246913580247, "rougeL_stderr": 0.011283149325161083 }, "e_tests.prompt2_qa_sk:biologia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.019047619047619046, "qf1_stderr": 0.019047619047619035, "rouge1": 0.02857142857142857, "rouge1_stderr": 0.01990372988626389, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.02857142857142857, "rougeL_stderr": 0.01990372988626389 }, "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { "acc": 0.3, "acc_stderr": 0.046056618647183814, "em_relaxed": 0.3, "em_relaxed_stderr": 0.046056618647183814 }, "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { "acc": 0.27, "acc_stderr": 0.044619604333847394, "em_relaxed": 0.27, "em_relaxed_stderr": 0.044619604333847394 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { "acc": 0.2, "acc_stderr": 0.032769276820761584, "em_relaxed": 0.20666666666666667, "em_relaxed_stderr": 0.033171868444173455 }, "slovak_tests.prompt2_qa_sk:monitor9|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "scio.prompt2_mqa_single_correct_sk:mat|0": { "acc": 0.23770491803278687, "acc_stderr": 0.022280975973791882, "em_relaxed": 0.24316939890710382, "em_relaxed_stderr": 0.022454705846132374 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { "acc": 0.18686868686868688, "acc_stderr": 0.027772533334218974, "em_relaxed": 0.1919191919191919, "em_relaxed_stderr": 0.028057791672989024 }, "multi_wiki_qa.prompt1_qa_sk|0": { "qem": 0.0756453872323394, "qem_stderr": 0.003741099997048089, "qf1": 0.12194744826062183, "qf1_stderr": 0.0038908205119327767 }, "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { "acc": 0.29, "acc_stderr": 0.045604802157206845, "em_relaxed": 0.29, "em_relaxed_stderr": 0.045604802157206845 }, "e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": { "acc": 0.03571428571428571, "acc_stderr": 0.03571428571428572 }, "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { "acc": 0.18018018018018017, "acc_stderr": 0.02585333527853839, "em_relaxed": 0.18018018018018017, "em_relaxed_stderr": 0.02585333527853839 }, "scio.prompt2_mqa_single_correct_sk:vsp|0": { "acc": 0.25849514563106796, "acc_stderr": 0.015261010021381638, "em_relaxed": 0.2621359223300971, "em_relaxed_stderr": 0.01533033114193304 }, "e_tests.prompt2_qa_sk:dejepis|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { "acc": 0.16580310880829016, "acc_stderr": 0.026839845022314415, "em_relaxed": 0.16580310880829016, "em_relaxed_stderr": 0.026839845022314415 }, "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { "acc": 0.21, "acc_stderr": 0.040936018074033256, "em_relaxed": 0.21, "em_relaxed_stderr": 0.040936018074033256 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { "acc": 0.23232323232323232, "acc_stderr": 0.04266016017054689, "em_relaxed": 0.24242424242424243, "em_relaxed_stderr": 0.04329004329004326 }, "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { "acc": 0.2463768115942029, "acc_stderr": 0.025984280443996907, "em_relaxed": 0.2463768115942029, "em_relaxed_stderr": 0.025984280443996907 }, "e_tests.prompt1_qa_sk:informatika|0": { "qem": 0.038461538461538464, "qem_stderr": 0.021915495473007307, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0032865282865282863, "rouge1_stderr": 0.0022973226212271926, "rouge2": 0.0008547008547008547, "rouge2_stderr": 0.0008547008547008546, "rougeL": 0.0032865282865282863, "rougeL_stderr": 0.0022973226212271926 }, "e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { "acc": 0.18, "acc_stderr": 0.038612291966536955, "em_relaxed": 0.18, "em_relaxed_stderr": 0.038612291966536955 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { "acc": 0.22119815668202766, "acc_stderr": 0.02824082382619082, "em_relaxed": 0.2304147465437788, "em_relaxed_stderr": 0.0286521112206854 }, "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { "acc": 0.22162883845126835, "acc_stderr": 0.015186430423507605, "em_relaxed": 0.22162883845126835, "em_relaxed_stderr": 0.015186430423507605 }, "slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": { "acc": 0.24744897959183673, "acc_stderr": 0.021823400829422215, "em_relaxed": 0.24744897959183673, "em_relaxed_stderr": 0.021823400829422215 }, "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { "acc": 0.3, "acc_stderr": 0.046056618647183814, "em_relaxed": 0.3, "em_relaxed_stderr": 0.046056618647183814 }, "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { "acc": 0.397172236503856, "acc_stderr": 0.01755397820008727, "em_relaxed": 0.40359897172236503, "em_relaxed_stderr": 0.017600852681572653 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { "acc": 0.24561403508771928, "acc_stderr": 0.057521390564052664, "em_relaxed": 0.24561403508771928, "em_relaxed_stderr": 0.057521390564052664 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { "acc": 0.21963824289405684, "acc_stderr": 0.021072124180856616, "em_relaxed": 0.21963824289405684, "em_relaxed_stderr": 0.021072124180856616 }, "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { "acc": 0.2803738317757009, "acc_stderr": 0.04362839933570099, "em_relaxed": 0.2803738317757009, "em_relaxed_stderr": 0.04362839933570099 }, "e_tests.prompt1_mqa_single_correct_sk:geografia|0": { "acc": 0.24489795918367346, "acc_stderr": 0.062069005411206336, "em_relaxed": 0.24489795918367346, "em_relaxed_stderr": 0.062069005411206336 }, "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { "acc": 0.13624678663239073, "acc_stderr": 0.012306865740559735, "em_relaxed": 0.14524421593830333, "em_relaxed_stderr": 0.012640374409743442 }, "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { "acc": 0.21739130434782608, "acc_stderr": 0.024872937110754584, "em_relaxed": 0.21739130434782608, "em_relaxed_stderr": 0.024872937110754584 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { "acc": 0.24444444444444444, "acc_stderr": 0.03212157057535212, "em_relaxed": 0.24444444444444444, "em_relaxed_stderr": 0.03212157057535212 }, "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { "acc": 0.22830440587449932, "acc_stderr": 0.015347207245326627, "em_relaxed": 0.22963951935914553, "em_relaxed_stderr": 0.015378696047536682 }, "e_tests.prompt3_qa_sk:informatika|0": { "qem": 0.02564102564102564, "qem_stderr": 0.01801282495677459, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0036324786324786326, "rouge1_stderr": 0.0025563439430967027, "rouge2": 0.0008547008547008547, "rouge2_stderr": 0.0008547008547008546, "rougeL": 0.0036324786324786326, "rougeL_stderr": 0.0025563439430967027 }, "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { "acc": 0.21753246753246752, "acc_stderr": 0.023546481379339165, "em_relaxed": 0.21753246753246752, "em_relaxed_stderr": 0.023546481379339165 }, "mmlu.prompt1_mqa_single_correct_sk:management|0": { "acc": 0.25, "acc_stderr": 0.04351941398892446, "em_relaxed": 0.25, "em_relaxed_stderr": 0.04351941398892446 }, "scio.prompt3_mqa_single_correct_sk:bio|0": { "acc": 0.24778761061946902, "acc_stderr": 0.023482897631940607, "em_relaxed": 0.26548672566371684, "em_relaxed_stderr": 0.024019440114877274 }, "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { "acc": 0.30275229357798167, "acc_stderr": 0.04421049631437605, "em_relaxed": 0.30275229357798167, "em_relaxed_stderr": 0.04421049631437605 }, "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { "acc": 0.42287917737789205, "acc_stderr": 0.017722745992252577, "em_relaxed": 0.42287917737789205, "em_relaxed_stderr": 0.017722745992252577 }, "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { "acc": 0.17647058823529413, "acc_stderr": 0.0379328118530781, "em_relaxed": 0.18627450980392157, "em_relaxed_stderr": 0.03873958714149352 }, "financial_exams.prompt1_completion_sk|0": { "acc": 0.18290854572713644, "acc_stderr": 0.010588568678863373, "em_relaxed": 0.18290854572713644, "em_relaxed_stderr": 0.010588568678863373 }, "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { "acc": 0.3211009174311927, "acc_stderr": 0.044927422774799206, "em_relaxed": 0.3211009174311927, "em_relaxed_stderr": 0.044927422774799206 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { "acc": 0.20666666666666667, "acc_stderr": 0.033171868444173455, "em_relaxed": 0.22, "em_relaxed_stderr": 0.033936379482039564 }, "sklep.prompt3_hate_speech_sk:hate-speech|0": { "acc": 0.0932524639878696, "acc_stderr": 0.00800968883832857, "em_norm_relaxed": 0.09401061410159212, "em_norm_relaxed_stderr": 0.008038819818872478 }, "e_tests.prompt3_qa_sk:geografia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { "acc": 0.233502538071066, "acc_stderr": 0.030218515643241007, "em_relaxed": 0.233502538071066, "em_relaxed_stderr": 0.030218515643241007 }, "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { "acc": 0.21212121212121213, "acc_stderr": 0.041296069325408906, "em_relaxed": 0.21212121212121213, "em_relaxed_stderr": 0.041296069325408906 }, "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { "acc": 0.23232323232323232, "acc_stderr": 0.042660160170546885, "em_relaxed": 0.23232323232323232, "em_relaxed_stderr": 0.042660160170546885 }, "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { "acc": 0.26785714285714285, "acc_stderr": 0.04203277291467762, "em_relaxed": 0.2767857142857143, "em_relaxed_stderr": 0.04246624336697624 }, "slovakcopa.prompt3_copa_sk|0": { "acc": 0.016, "acc_stderr": 0.0056170367814719815, "em_relaxed": 0.016, "em_relaxed_stderr": 0.0056170367814719815 }, "e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": { "acc": 0.02142857142857143, "acc_stderr": 0.012282463299882659 }, "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { "acc": 0.27102803738317754, "acc_stderr": 0.043172737765666686, "em_relaxed": 0.27102803738317754, "em_relaxed_stderr": 0.043172737765666686 }, "slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": { "acc": 0.2903225806451613, "acc_stderr": 0.030884745411494743, "em_relaxed": 0.2903225806451613, "em_relaxed_stderr": 0.030884745411494743 }, "financial_exams.prompt2_completion_sk|0": { "acc": 0.1896551724137931, "acc_stderr": 0.010737475929667924, "em_relaxed": 0.1904047976011994, "em_relaxed_stderr": 0.010753697898553788 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { "acc": 0.19170984455958548, "acc_stderr": 0.02840895362624528, "em_relaxed": 0.19170984455958548, "em_relaxed_stderr": 0.02840895362624528 }, "e_tests.prompt1_qa_sk:slovensky-jazyk|0": { "qem": 0.0051813471502590676, "qem_stderr": 0.00518134715025908, "qf1": 0.01526314790563495, "qf1_stderr": 0.006730183842906608, "rouge1": 0.01724455022545386, "rouge1_stderr": 0.006851608530741241, "rouge2": 0.0073447868375602215, "rouge2_stderr": 0.005570575389175516, "rougeL": 0.01724455022545386, "rougeL_stderr": 0.006851608530741241 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { "acc": 0.22962962962962963, "acc_stderr": 0.03633384414073462, "em_relaxed": 0.22962962962962963, "em_relaxed_stderr": 0.03633384414073462 }, "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { "acc": 0.2862644415917843, "acc_stderr": 0.01620550507370852, "em_relaxed": 0.28754813863928114, "em_relaxed_stderr": 0.01622718713515171 }, "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { "acc": 0.2537313432835821, "acc_stderr": 0.037731914327133756, "em_relaxed": 0.2537313432835821, "em_relaxed_stderr": 0.037731914327133756 }, "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { "acc": 0.04702495201535509, "acc_stderr": 0.006561144166938836, "em_norm_relaxed": 0.05566218809980806, "em_norm_relaxed_stderr": 0.007105889705516419 }, "toxic_sk.prompt3_toxicity_sk|0": { "acc": 0.007945516458569807, "acc_stderr": 0.0029928675111869866, "em_norm_relaxed": 0.007945516458569807, "em_norm_relaxed_stderr": 0.0029928675111869866 }, "e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": { "acc": 0.017857142857142856, "acc_stderr": 0.017857142857142863 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { "acc": 0.24561403508771928, "acc_stderr": 0.05752139056405266, "em_relaxed": 0.24561403508771928, "em_relaxed_stderr": 0.05752139056405266 }, "slovak_tests.prompt3_qa_sk:maturita|0": { "qem": 0.01327433628318584, "qem_stderr": 0.007629803632081178, "qf1": 0.011971484759095378, "qf1_stderr": 0.0058019008534905304, "rouge1": 0.014379696684114138, "rouge1_stderr": 0.0046830153944730545, "rouge2": 0.004080629301868239, "rouge2_stderr": 0.002569082757498883, "rougeL": 0.014379696684114138, "rougeL_stderr": 0.0046830153944730545 }, "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { "acc": 0.2, "acc_stderr": 0.04020151261036845, "em_relaxed": 0.2, "em_relaxed_stderr": 0.04020151261036845 }, "slovak_tests.prompt2_qa_sk:maturita|0": { "qem": 0.008849557522123894, "qem_stderr": 0.0062436609110668876, "qf1": 0.0057030481809242865, "qf1_stderr": 0.0035655859886822355, "rouge1": 0.015324943300924803, "rouge1_stderr": 0.0047498192758248054, "rouge2": 0.0022123893805309734, "rouge2_stderr": 0.002212389380530974, "rougeL": 0.015324943300924803, "rougeL_stderr": 0.0047498192758248054 }, "slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.036387264457439894, "qf1_stderr": 0.025966525279646633, "rouge1": 0.027777777777777776, "rouge1_stderr": 0.019641855032959652, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.027777777777777776, "rougeL_stderr": 0.019641855032959652 }, "e_tests.prompt2_mqa_single_correct_sk:dejepis|0": { "acc": 0.1875, "acc_stderr": 0.05693291124629694, "em_relaxed": 0.1875, "em_relaxed_stderr": 0.05693291124629694 }, "slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": { "acc": 0.2653061224489796, "acc_stderr": 0.022327416188566384, "em_relaxed": 0.2653061224489796, "em_relaxed_stderr": 0.022327416188566384 }, "e_tests.prompt2_qa_sk:chemia|0": { "qem": 0.037037037037037035, "qem_stderr": 0.025940896147128693, "qf1": 0.012345679012345678, "qf1_stderr": 0.012345679012345675, "rouge1": 0.02580246913580247, "rouge1_stderr": 0.011283149325161083, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.02580246913580247, "rougeL_stderr": 0.011283149325161083 }, "e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": { "acc": 0.26419213973799127, "acc_stderr": 0.020624529862085718, "em_relaxed": 0.26419213973799127, "em_relaxed_stderr": 0.020624529862085718 }, "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { "acc": 0.22402597402597402, "acc_stderr": 0.02379597980086175, "em_relaxed": 0.22402597402597402, "em_relaxed_stderr": 0.02379597980086175 }, "e_tests.prompt1_mqa_single_correct_sk:fyzika|0": { "acc": 0.22727272727272727, "acc_stderr": 0.0914486154730632, "em_relaxed": 0.22727272727272727, "em_relaxed_stderr": 0.0914486154730632 }, "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { "acc": 0.19318181818181818, "acc_stderr": 0.024344064423295675, "em_relaxed": 0.19318181818181818, "em_relaxed_stderr": 0.024344064423295675 }, "e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": { "acc": 0.1, "acc_stderr": 0.06882472016116853 }, "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { "acc": 0.29, "acc_stderr": 0.045604802157206845, "em_relaxed": 0.3, "em_relaxed_stderr": 0.046056618647183814 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { "acc": 0.24812030075187969, "acc_stderr": 0.02653275776026344, "em_relaxed": 0.2518796992481203, "em_relaxed_stderr": 0.026666091559782525 }, "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { "acc": 0.2605633802816901, "acc_stderr": 0.03696558732615874, "em_relaxed": 0.2605633802816901, "em_relaxed_stderr": 0.03696558732615874 }, "e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": { "acc": 0.02857142857142857, "acc_stderr": 0.028571428571428574 }, "slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.024691358024691357, "qf1_stderr": 0.024691358024691353, "rouge1": 0.01851851851851852, "rouge1_stderr": 0.01851851851851852, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.01851851851851852, "rougeL_stderr": 0.01851851851851852 }, "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { "acc": 0.23134328358208955, "acc_stderr": 0.036565274697599, "em_relaxed": 0.23134328358208955, "em_relaxed_stderr": 0.036565274697599 }, "mmlu.prompt3_mqa_single_correct_sk:management|0": { "acc": 0.26, "acc_stderr": 0.04408440022768078, "em_relaxed": 0.26, "em_relaxed_stderr": 0.04408440022768078 }, "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { "acc": 0.2088607594936709, "acc_stderr": 0.03244180901572427, "em_relaxed": 0.2088607594936709, "em_relaxed_stderr": 0.03244180901572427 }, "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { "acc": 0.31, "acc_stderr": 0.04648231987117316, "em_relaxed": 0.31, "em_relaxed_stderr": 0.04648231987117316 }, "scio.prompt2_mqa_single_correct_sk:aj|0": { "acc": 0.3, "acc_stderr": 0.15275252316519464, "em_relaxed": 0.3, "em_relaxed_stderr": 0.15275252316519464 }, "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { "acc": 0.21739130434782608, "acc_stderr": 0.038631448549506, "em_relaxed": 0.25217391304347825, "em_relaxed_stderr": 0.04067222754154718 }, "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { "acc": 0.2457627118644068, "acc_stderr": 0.03980329854920433, "em_relaxed": 0.2457627118644068, "em_relaxed_stderr": 0.03980329854920433 }, "e_tests.prompt1_mqa_single_correct_sk:chemia|0": { "acc": 0.17857142857142858, "acc_stderr": 0.05164277182008721, "em_relaxed": 0.17857142857142858, "em_relaxed_stderr": 0.05164277182008721 }, "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { "acc": 0.2386058981233244, "acc_stderr": 0.022099063590345355, "em_relaxed": 0.2386058981233244, "em_relaxed_stderr": 0.022099063590345355 }, "e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": { "acc": 0.27510917030567683, "acc_stderr": 0.020889629452003455, "em_relaxed": 0.27510917030567683, "em_relaxed_stderr": 0.020889629452003455 }, "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { "acc": 0.17763157894736842, "acc_stderr": 0.03110318238312337, "em_relaxed": 0.18421052631578946, "em_relaxed_stderr": 0.031546980450822305 }, "sklep.prompt1_hate_speech_sk:hate-speech|0": { "acc": 0.3525398028809704, "acc_stderr": 0.01315990975593033, "em_norm_relaxed": 0.35329795299469297, "em_norm_relaxed_stderr": 0.013166337192115686 }, "e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { "acc": 0.21641791044776118, "acc_stderr": 0.035707794739409195, "em_relaxed": 0.22388059701492538, "em_relaxed_stderr": 0.03614487092173135 }, "e_tests.prompt1_mqa_single_correct_sk:matematika|0": { "acc": 0.1282051282051282, "acc_stderr": 0.05423355275954149, "em_relaxed": 0.1282051282051282, "em_relaxed_stderr": 0.05423355275954149 }, "sklep.prompt3_qa_sk:question-answering|0": { "qem": 0.05123656475007826, "qem_stderr": 0.0022523759848186195, "qf1": 0.1417299858339243, "qf1_stderr": 0.002706574024592746 }, "e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": { "acc": 0.022727272727272728, "acc_stderr": 0.022727272727272728 }, "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { "acc": 0.2077922077922078, "acc_stderr": 0.023156077580247444, "em_relaxed": 0.2077922077922078, "em_relaxed_stderr": 0.023156077580247444 }, "toxic_sk.prompt2_toxicity_sk|0": { "acc": 0.23609534619750283, "acc_stderr": 0.014316008748210712, "em_norm_relaxed": 0.23723041997729852, "em_norm_relaxed_stderr": 0.014339715502704882 }, "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { "acc": 0.24770642201834864, "acc_stderr": 0.041538463936410155, "em_relaxed": 0.24770642201834864, "em_relaxed_stderr": 0.041538463936410155 }, "e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": { "acc": 0.03571428571428571, "acc_stderr": 0.03571428571428572 }, "senti_sk.prompt3_sentiment_binary_sk|0": { "acc": 0.7416845493562232, "acc_stderr": 0.007169768138157722, "em_norm_relaxed": 0.7475858369098712, "em_norm_relaxed_stderr": 0.007115537241408355 }, "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { "acc": 0.22758620689655173, "acc_stderr": 0.03493950380131184, "em_relaxed": 0.22758620689655173, "em_relaxed_stderr": 0.03493950380131184 }, "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { "acc": 0.1856060606060606, "acc_stderr": 0.023973722216817445, "em_relaxed": 0.1856060606060606, "em_relaxed_stderr": 0.023973722216817445 }, "e_tests.prompt2_mqa_single_correct_sk:informatika|0": { "acc": 0.232, "acc_stderr": 0.03790650637827825, "em_relaxed": 0.232, "em_relaxed_stderr": 0.03790650637827825 }, "mmlu.prompt2_mqa_single_correct_sk:management|0": { "acc": 0.27, "acc_stderr": 0.044619604333847394, "em_relaxed": 0.27, "em_relaxed_stderr": 0.044619604333847394 }, "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { "acc": 0.27, "acc_stderr": 0.044619604333847394, "em_relaxed": 0.27, "em_relaxed_stderr": 0.044619604333847394 }, "toxic_sk.prompt1_toxicity_sk|0": { "acc": 0.3530079455164586, "acc_stderr": 0.01611017944424598, "em_norm_relaxed": 0.35414301929625425, "em_norm_relaxed_stderr": 0.016121898615772852 }, "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { "acc": 0.2651006711409396, "acc_stderr": 0.018095094525450207, "em_relaxed": 0.26677852348993286, "em_relaxed_stderr": 0.018131533555282176 }, "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { "acc": 0.23448275862068965, "acc_stderr": 0.035306258743465914, "em_relaxed": 0.23448275862068965, "em_relaxed_stderr": 0.035306258743465914 }, "sklep.prompt2_hate_speech_sk:hate-speech|0": { "acc": 0.1956027293404094, "acc_stderr": 0.010926096810556464, "em_norm_relaxed": 0.1956027293404094, "em_norm_relaxed_stderr": 0.010926096810556464 }, "scio.prompt3_mqa_single_correct_sk:che|0": { "acc": 0.24796747967479674, "acc_stderr": 0.027588788664854164, "em_relaxed": 0.24796747967479674, "em_relaxed_stderr": 0.027588788664854164 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { "acc": 0.21717171717171718, "acc_stderr": 0.029376616484945637, "em_relaxed": 0.21717171717171718, "em_relaxed_stderr": 0.029376616484945637 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { "acc": 0.2222222222222222, "acc_stderr": 0.031073844843659416, "em_relaxed": 0.2222222222222222, "em_relaxed_stderr": 0.031073844843659416 }, "e_tests.prompt1_mqa_single_correct_sk:biologia|0": { "acc": 0.3103448275862069, "acc_stderr": 0.04988718850038745, "em_relaxed": 0.3103448275862069, "em_relaxed_stderr": 0.04988718850038745 }, "e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": { "acc": 0.0, "acc_stderr": 0.0 }, "e_tests.prompt3_mqa_single_correct_sk:chemia|0": { "acc": 0.23214285714285715, "acc_stderr": 0.05692939024000109, "em_relaxed": 0.23214285714285715, "em_relaxed_stderr": 0.05692939024000109 }, "e_tests.prompt2_mqa_single_correct_sk:chemia|0": { "acc": 0.21428571428571427, "acc_stderr": 0.055328333517248834, "em_relaxed": 0.21428571428571427, "em_relaxed_stderr": 0.055328333517248834 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { "acc": 0.17616580310880828, "acc_stderr": 0.027493504244548047, "em_relaxed": 0.17616580310880828, "em_relaxed_stderr": 0.027493504244548047 }, "e_tests.prompt3_mqa_single_correct_sk:matematika|0": { "acc": 0.20512820512820512, "acc_stderr": 0.06550424345215436, "em_relaxed": 0.20512820512820512, "em_relaxed_stderr": 0.06550424345215436 }, "e_tests.prompt3_qa_sk:obcianska-nauka|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.020408163265306124, "rouge1_stderr": 0.020408163265306124, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.020408163265306124, "rougeL_stderr": 0.020408163265306124 }, "e_tests.prompt3_mqa_single_correct_sk:dejepis|0": { "acc": 0.14583333333333334, "acc_stderr": 0.05148145136372286, "em_relaxed": 0.14583333333333334, "em_relaxed_stderr": 0.05148145136372286 }, "e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": { "acc": 0.022727272727272728, "acc_stderr": 0.022727272727272728 }, "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { "acc": 0.25704526046114434, "acc_stderr": 0.012775945430940199, "em_relaxed": 0.2681468830059778, "em_relaxed_stderr": 0.012951062793076921 }, "sklep.prompt2_nli_3_classes_sk:nli|0": { "acc": 0.04936051159072742, "acc_stderr": 0.003062545148658699, "em_norm_relaxed": 0.06474820143884892, "em_norm_relaxed_stderr": 0.003479067694461015 }, "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { "acc": 0.2088607594936709, "acc_stderr": 0.03244180901572427, "em_relaxed": 0.2088607594936709, "em_relaxed_stderr": 0.03244180901572427 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { "acc": 0.2332089552238806, "acc_stderr": 0.01828244336455249, "em_relaxed": 0.23507462686567165, "em_relaxed_stderr": 0.01833308365747139 }, "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { "acc": 0.26605504587155965, "acc_stderr": 0.042521210223474465, "em_relaxed": 0.27522935779816515, "em_relaxed_stderr": 0.04297697040536756 }, "senti_sk.prompt2_sentiment_3_class_sk|0": { "acc": 0.3158007269617276, "acc_stderr": 0.006797678423770271, "em_norm_relaxed": 0.32050459696386574, "em_norm_relaxed_stderr": 0.006824536198921628 }, "slovak_tests.prompt1_qa_sk:monitor9|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { "acc": 0.20833333333333334, "acc_stderr": 0.02504221193381759, "em_relaxed": 0.20833333333333334, "em_relaxed_stderr": 0.02504221193381759 }, "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { "acc": 0.24107142857142858, "acc_stderr": 0.04059867246952688, "em_relaxed": 0.25892857142857145, "em_relaxed_stderr": 0.041577515398656284 }, "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { "acc": 0.24647887323943662, "acc_stderr": 0.03629342967383697, "em_relaxed": 0.24647887323943662, "em_relaxed_stderr": 0.03629342967383697 }, "e_tests.prompt3_qa_sk:biologia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.019047619047619046, "qf1_stderr": 0.019047619047619035, "rouge1": 0.014285714285714285, "rouge1_stderr": 0.014285714285714287, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.014285714285714285, "rougeL_stderr": 0.014285714285714287 }, "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { "acc": 0.25705329153605017, "acc_stderr": 0.02450624980268527, "em_relaxed": 0.26332288401253917, "em_relaxed_stderr": 0.024698429072332967 }, "e_tests.prompt3_mqa_single_correct_sk:fyzika|0": { "acc": 0.18181818181818182, "acc_stderr": 0.08416546361568647, "em_relaxed": 0.18181818181818182, "em_relaxed_stderr": 0.08416546361568647 }, "senti_sk.prompt1_sentiment_3_class_sk|0": { "acc": 0.25486422920675644, "acc_stderr": 0.006372867628034562, "em_norm_relaxed": 0.2572161642078255, "em_norm_relaxed_stderr": 0.00639209321447505 }, "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { "acc": 0.3, "acc_stderr": 0.046056618647183814, "em_relaxed": 0.3, "em_relaxed_stderr": 0.046056618647183814 }, "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { "acc": 0.16666666666666666, "acc_stderr": 0.03708284662416542, "em_relaxed": 0.16666666666666666, "em_relaxed_stderr": 0.03708284662416542 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { "acc": 0.25806451612903225, "acc_stderr": 0.029772845386803175, "em_relaxed": 0.25806451612903225, "em_relaxed_stderr": 0.029772845386803175 }, "scio.prompt2_mqa_single_correct_sk:bio|0": { "acc": 0.23008849557522124, "acc_stderr": 0.02289335836732507, "em_relaxed": 0.2448377581120944, "em_relaxed_stderr": 0.02338842519033371 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { "acc": 0.18691588785046728, "acc_stderr": 0.02671167024582658, "em_relaxed": 0.19158878504672897, "em_relaxed_stderr": 0.026965681762486853 }, "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { "acc": 0.2711864406779661, "acc_stderr": 0.041100705493392085, "em_relaxed": 0.2711864406779661, "em_relaxed_stderr": 0.041100705493392085 }, "e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": { "acc": 0.18840579710144928, "acc_stderr": 0.047420064740574225, "em_relaxed": 0.18840579710144928, "em_relaxed_stderr": 0.047420064740574225 }, "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { "acc": 0.24092409240924093, "acc_stderr": 0.0246081660183802, "em_relaxed": 0.24752475247524752, "em_relaxed_stderr": 0.0248343018351657 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { "acc": 0.2727272727272727, "acc_stderr": 0.03173071239071724, "em_relaxed": 0.2727272727272727, "em_relaxed_stderr": 0.03173071239071724 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { "acc": 0.23255813953488372, "acc_stderr": 0.021502792400356758, "em_relaxed": 0.23255813953488372, "em_relaxed_stderr": 0.021502792400356758 }, "scio.prompt1_mqa_single_correct_sk:che|0": { "acc": 0.27235772357723576, "acc_stderr": 0.028441059280558723, "em_relaxed": 0.27235772357723576, "em_relaxed_stderr": 0.028441059280558723 }, "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { "acc": 0.18604651162790697, "acc_stderr": 0.029758600618213775, "em_relaxed": 0.18604651162790697, "em_relaxed_stderr": 0.029758600618213775 }, "slovak_tests.prompt3_qa_sk:monitor9|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { "acc": 0.21481481481481482, "acc_stderr": 0.03547854198560827, "em_relaxed": 0.21481481481481482, "em_relaxed_stderr": 0.03547854198560827 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { "acc": 0.19696969696969696, "acc_stderr": 0.028335609732463348, "em_relaxed": 0.19696969696969696, "em_relaxed_stderr": 0.028335609732463348 }, "e_tests.prompt1_mqa_single_correct_sk:informatika|0": { "acc": 0.24, "acc_stderr": 0.03835319728439192, "em_relaxed": 0.24, "em_relaxed_stderr": 0.03835319728439192 }, "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { "acc": 0.28085106382978725, "acc_stderr": 0.029379170464124825, "em_relaxed": 0.28085106382978725, "em_relaxed_stderr": 0.029379170464124825 }, "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { "acc": 0.23102310231023102, "acc_stderr": 0.02425386079112883, "em_relaxed": 0.23102310231023102, "em_relaxed_stderr": 0.02425386079112883 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { "acc": 0.22695035460992907, "acc_stderr": 0.024987106365642973, "em_relaxed": 0.23049645390070922, "em_relaxed_stderr": 0.025123739226872405 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { "acc": 0.23333333333333334, "acc_stderr": 0.03161296043612007, "em_relaxed": 0.23333333333333334, "em_relaxed_stderr": 0.03161296043612007 }, "sklep.prompt1_nli_binary_sk:rte|0": { "acc": 0.26385542168674697, "acc_stderr": 0.010820351700856653, "em_norm_relaxed": 0.26626506024096386, "em_norm_relaxed_stderr": 0.01085184289579353 }, "slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": { "acc": 0.22959183673469388, "acc_stderr": 0.02126915716826347, "em_relaxed": 0.22959183673469388, "em_relaxed_stderr": 0.02126915716826347 }, "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { "acc": 0.3486238532110092, "acc_stderr": 0.04585456701913343, "em_relaxed": 0.3486238532110092, "em_relaxed_stderr": 0.04585456701913343 }, "slohry.prompt1_inflection_sk:inflection_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_declension_sk:declension_high|0": { "acc": 0.0748502994011976, "acc_stderr": 0.014420511342862789, "em_norm_relaxed": 0.0748502994011976, "em_norm_relaxed_stderr": 0.014420511342862789 }, "slohry.prompt3_gradation_sk:gradation_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_inflection_sk:inflection_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_gradation_sk:gradation_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_declension_sk:declension_high|0": { "acc": 0.059880239520958084, "acc_stderr": 0.013002032231530652, "em_norm_relaxed": 0.059880239520958084, "em_norm_relaxed_stderr": 0.013002032231530652 }, "chiby.prompt2_chiby_sk|0": { "acc": 0.025349650349650348, "acc_stderr": 0.0026834904197041206, "em_norm_relaxed": 0.02564102564102564, "em_norm_relaxed_stderr": 0.002698465265697882 }, "slovak_trivia.prompt3_mqa_single_correct_sk|0": { "acc": 0.28611898016997167, "acc_stderr": 0.024088786371538086, "em_relaxed": 0.28611898016997167, "em_relaxed_stderr": 0.024088786371538086 }, "slohry.prompt1_inflection_sk:inflection_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_gradation_sk:gradation_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "chiby.prompt3_chiby_sk|0": { "acc": 0.1622960372960373, "acc_stderr": 0.00629490070363013, "em_norm_relaxed": 0.16258741258741258, "em_norm_relaxed_stderr": 0.00629945304550759 }, "slohry.prompt2_declension_sk:declension_mid|0": { "acc": 0.05389221556886228, "acc_stderr": 0.012374031301903447, "em_norm_relaxed": 0.05389221556886228, "em_norm_relaxed_stderr": 0.012374031301903447 }, "slohry.prompt2_gradation_sk:gradation_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_gradation_sk:gradation_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_identity_form_sk:identity_form_low|0": { "acc": 0.005988023952095809, "acc_stderr": 0.004227809946139441, "em_norm_relaxed": 0.005988023952095809, "em_norm_relaxed_stderr": 0.004227809946139441 }, "slohry.prompt3_declension_sk:declension_high|0": { "acc": 0.0718562874251497, "acc_stderr": 0.014152002095505148, "em_norm_relaxed": 0.0718562874251497, "em_norm_relaxed_stderr": 0.014152002095505148 }, "slohry.prompt3_declension_sk:declension_mid|0": { "acc": 0.0658682634730539, "acc_stderr": 0.013593148263003514, "em_norm_relaxed": 0.0658682634730539, "em_norm_relaxed_stderr": 0.013593148263003514 }, "slohry.prompt1_identity_form_sk:identity_form_high|0": { "acc": 0.0029940119760479044, "acc_stderr": 0.0029940119760478922, "em_norm_relaxed": 0.0029940119760479044, "em_norm_relaxed_stderr": 0.0029940119760478922 }, "slohry.prompt3_declension_sk:declension_low|0": { "acc": 0.04790419161676647, "acc_stderr": 0.011703209185183546, "em_norm_relaxed": 0.04790419161676647, "em_norm_relaxed_stderr": 0.011703209185183546 }, "slohry.prompt1_declension_sk:declension_mid|0": { "acc": 0.0718562874251497, "acc_stderr": 0.014152002095505158, "em_norm_relaxed": 0.0718562874251497, "em_norm_relaxed_stderr": 0.014152002095505158 }, "slohry.prompt3_inflection_sk:inflection_high|0": { "acc": 0.008982035928143712, "acc_stderr": 0.005170184492247012, "em_norm_relaxed": 0.008982035928143712, "em_norm_relaxed_stderr": 0.005170184492247012 }, "slohry.prompt2_declension_sk:declension_low|0": { "acc": 0.029940119760479042, "acc_stderr": 0.00933907625586894, "em_norm_relaxed": 0.029940119760479042, "em_norm_relaxed_stderr": 0.00933907625586894 }, "slohry.prompt3_gradation_sk:gradation_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_identity_form_sk:identity_form_low|0": { "acc": 0.008982035928143712, "acc_stderr": 0.005170184492247015, "em_norm_relaxed": 0.008982035928143712, "em_norm_relaxed_stderr": 0.005170184492247015 }, "slovak_trivia.prompt1_mqa_single_correct_sk|0": { "acc": 0.26628895184135976, "acc_stderr": 0.02355959235547647, "em_relaxed": 0.26628895184135976, "em_relaxed_stderr": 0.02355959235547647 }, "analogies_sk.prompt3_analogy_sk|0": { "acc": 0.0, "acc_stderr": 0.0 }, "slohry.prompt3_gradation_sk:gradation_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_inflection_sk:inflection_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_inflection_sk:inflection_high|0": { "acc": 0.0029940119760479044, "acc_stderr": 0.0029940119760478966, "em_norm_relaxed": 0.0029940119760479044, "em_norm_relaxed_stderr": 0.0029940119760478966 }, "slohry.prompt3_inflection_sk:inflection_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_identity_form_sk:identity_form_high|0": { "acc": 0.005988023952095809, "acc_stderr": 0.004227809946139438, "em_norm_relaxed": 0.005988023952095809, "em_norm_relaxed_stderr": 0.004227809946139438 }, "slohry.prompt2_identity_form_sk:identity_form_mid|0": { "acc": 0.02694610778443114, "acc_stderr": 0.008873487700057157, "em_norm_relaxed": 0.02694610778443114, "em_norm_relaxed_stderr": 0.008873487700057157 }, "slohry.prompt3_identity_form_sk:identity_form_mid|0": { "acc": 0.008982035928143712, "acc_stderr": 0.0051701844922470075, "em_norm_relaxed": 0.008982035928143712, "em_norm_relaxed_stderr": 0.0051701844922470075 }, "slohry.prompt3_identity_form_sk:identity_form_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_declension_sk:declension_low|0": { "acc": 0.05089820359281437, "acc_stderr": 0.012044409374838807, "em_norm_relaxed": 0.05089820359281437, "em_norm_relaxed_stderr": 0.012044409374838807 }, "slohry.prompt1_gradation_sk:gradation_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "analogies_sk.prompt2_analogy_sk|0": { "acc": 0.0007544322897019992, "acc_stderr": 0.0005333635248765946 }, "analogies_sk.prompt1_analogy_sk|0": { "acc": 0.001131648434552999, "acc_stderr": 0.0006531109317700804 }, "slohry.prompt1_gradation_sk:gradation_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_identity_form_sk:identity_form_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_inflection_sk:inflection_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_inflection_sk:inflection_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "chiby.prompt1_chiby_sk|0": { "acc": 0.23572261072261072, "acc_stderr": 0.00724629095400738, "em_norm_relaxed": 0.23892773892773891, "em_norm_relaxed_stderr": 0.007280075348854293 }, "slovak_trivia.prompt2_mqa_single_correct_sk|0": { "acc": 0.28328611898017, "acc_stderr": 0.024016749232030024, "em_relaxed": 0.28328611898017, "em_relaxed_stderr": 0.024016749232030024 }, "slohry.prompt1_identity_form_sk:identity_form_mid|0": { "acc": 0.011976047904191617, "acc_stderr": 0.005960989845848331, "em_norm_relaxed": 0.011976047904191617, "em_norm_relaxed_stderr": 0.005960989845848331 }, "movie_review_SK_AVERAGE|0": { "em_norm_relaxed": 0.14890438247011953, "acc": 0.1457503320053121, "em_norm_relaxed_stderr": 0.007823268108665498, "acc_stderr": 0.007746689156547597, "em_norm_relaxed__prompts": { "1": 0.10109561752988047, "3": 0.20916334661354583, "2": 0.13645418326693226 }, "acc__prompts": { "1": 0.09860557768924302, "3": 0.20766932270916336, "2": 0.13097609561752988 } }, "movie_review_AVERAGE|0": { "em_norm_relaxed": 0.14890438247011953, "em_norm_relaxed_stderr": 0.007823268108665498, "acc": 0.1457503320053121, "acc_stderr": 0.007746689156547597 }, "e_tests_mqa_SK_AVERAGE|0": { "em_relaxed": 0.24309199020636585, "acc": 0.18150087260034903, "em_relaxed_stderr": 0.03621678600189384, "acc_stderr": 0.03214832055891622, "em_relaxed__prompts": { "2": 0.24239244491080797, "1": 0.2497376705141658, "3": 0.23714585519412382 }, "acc__prompts": { "2": 0.18175018698578907, "1": 0.1862378459237098, "3": 0.17651458489154825 } }, "e_tests_mqa_AVERAGE|0": { "em_relaxed": 0.24309199020636585, "em_relaxed_stderr": 0.03621678600189384, "acc": 0.18150087260034903, "acc_stderr": 0.03214832055891622 }, "mmlu_SK_AVERAGE|0": { "em_relaxed": 0.24332669831062115, "acc": 0.24148930740570612, "em_relaxed_stderr": 0.0266013460946822, "acc_stderr": 0.026529494985304103, "em_relaxed__prompts": { "2": 0.24628693921298422, "3": 0.2474353085285561, "1": 0.23625784719032308 }, "acc__prompts": { "2": 0.2440667585362119, "3": 0.24437299035369775, "1": 0.23602817332720868 } }, "mmlu_AVERAGE|0": { "em_relaxed": 0.24332669831062115, "em_relaxed_stderr": 0.0266013460946822, "acc": 0.24148930740570612, "acc_stderr": 0.026529494985304103 }, "belebele_SK_AVERAGE|0": { "em_relaxed": 0.267037037037037, "acc": 0.26666666666666666, "em_relaxed_stderr": 0.014753176443555857, "acc_stderr": 0.014746835660938531, "em_relaxed__prompts": { "3": 0.2722222222222222, "1": 0.2577777777777778, "2": 0.27111111111111114 }, "acc__prompts": { "3": 0.27111111111111114, "1": 0.2577777777777778, "2": 0.27111111111111114 } }, "belebele_AVERAGE|0": { "em_relaxed": 0.267037037037037, "em_relaxed_stderr": 0.014753176443555857, "acc": 0.26666666666666666, "acc_stderr": 0.014746835660938531 }, "senti_sk_binary_SK_AVERAGE|0": { "em_norm_relaxed": 0.49356223175965663, "acc": 0.4873927038626609, "em_norm_relaxed_stderr": 0.007613926970563059, "acc_stderr": 0.0076136774621452415, "em_norm_relaxed__prompts": { "2": 0.3261802575107296, "1": 0.4069206008583691, "3": 0.7475858369098712 }, "acc__prompts": { "2": 0.32135193133047213, "1": 0.39914163090128757, "3": 0.7416845493562232 } }, "senti_sk_binary_AVERAGE|0": { "em_norm_relaxed": 0.49356223175965663, "em_norm_relaxed_stderr": 0.007613926970563059, "acc": 0.4873927038626609, "acc_stderr": 0.0076136774621452415 }, "slovak_tests_mqa_SK_AVERAGE|0": { "em_relaxed": 0.2600800246229609, "acc": 0.2600800246229609, "em_relaxed_stderr": 0.02573346982142187, "acc_stderr": 0.02573346982142187, "em_relaxed__prompts": { "2": 0.26685133887349954, "3": 0.25761772853185594, "1": 0.25577100646352724 }, "acc__prompts": { "2": 0.26685133887349954, "3": 0.25761772853185594, "1": 0.25577100646352724 } }, "slovak_tests_mqa_AVERAGE|0": { "em_relaxed": 0.2600800246229609, "em_relaxed_stderr": 0.02573346982142187, "acc": 0.2600800246229609, "acc_stderr": 0.02573346982142187 }, "e_tests_qa_SK_AVERAGE|0": { "rouge2": 0.003228194982976972, "rouge1": 0.014818769352489746, "rougeL": 0.014818769352489746, "qf1": 0.007878222192390297, "qem": 0.012039127163280662, "rouge2_stderr": 0.002547306056000719, "rouge1_stderr": 0.008024257279824093, "rougeL_stderr": 0.008024257279824093, "qf1_stderr": 0.005529975861330393, "qem_stderr": 0.00874750521016363, "rouge2__prompts": { "3": 0.0030234626171420755, "2": 0.003310759969902182, "1": 0.0033503623618866578 }, "rouge1__prompts": { "3": 0.013608596069092682, "2": 0.014902802239145354, "1": 0.015944909749231198 }, "rougeL__prompts": { "3": 0.013608596069092682, "2": 0.014902802239145354, "1": 0.015944909749231198 }, "qf1__prompts": { "3": 0.005311379630859115, "2": 0.008663871869289476, "1": 0.009659415077022301 }, "qem__prompts": { "3": 0.011286681715575621, "2": 0.013544018058690745, "1": 0.011286681715575621 } }, "e_tests_qa_AVERAGE|0": { "rouge2": 0.003228194982976972, "rouge2_stderr": 0.002547306056000719, "rouge1": 0.014818769352489746, "rouge1_stderr": 0.008024257279824093, "rougeL": 0.014818769352489746, "rougeL_stderr": 0.008024257279824093, "qf1": 0.007878222192390297, "qf1_stderr": 0.005529975861330393, "qem": 0.012039127163280662, "qem_stderr": 0.00874750521016363 }, "scio_SK_AVERAGE|0": { "em_relaxed": 0.25434173669467786, "acc": 0.250046685340803, "em_relaxed_stderr": 0.02074943266589043, "acc_stderr": 0.020633005498944187, "em_relaxed__prompts": { "3": 0.27675070028011206, "2": 0.2549019607843137, "1": 0.23137254901960785 }, "acc__prompts": { "3": 0.2700280112044818, "2": 0.24929971988795518, "1": 0.230812324929972 } }, "scio_AVERAGE|0": { "em_relaxed": 0.25434173669467786, "em_relaxed_stderr": 0.02074943266589043, "acc": 0.250046685340803, "acc_stderr": 0.020633005498944187 }, "arc_SK_AVERAGE|0": { "em_relaxed": 0.24664955802680352, "acc": 0.2446535500427716, "em_relaxed_stderr": 0.012612937670503793, "acc_stderr": 0.012578166445440875, "em_relaxed__prompts": { "2": 0.24721984602224123, "3": 0.24550898203592814, "1": 0.24721984602224123 }, "acc__prompts": { "2": 0.24123182207014543, "3": 0.24550898203592814, "1": 0.24721984602224123 } }, "arc_AVERAGE|0": { "em_relaxed": 0.24664955802680352, "em_relaxed_stderr": 0.012612937670503793, "acc": 0.2446535500427716, "acc_stderr": 0.012578166445440875 }, "sklep_rte_SK_AVERAGE|0": { "em_norm_relaxed": 0.31506024096385543, "acc": 0.3140562248995984, "em_norm_relaxed_stderr": 0.010750305246379226, "acc_stderr": 0.010735367420948657, "em_norm_relaxed__prompts": { "3": 0.16566265060240964, "2": 0.5132530120481927, "1": 0.26626506024096386 }, "acc__prompts": { "3": 0.1650602409638554, "2": 0.5132530120481927, "1": 0.26385542168674697 } }, "sklep_rte_AVERAGE|0": { "em_norm_relaxed": 0.31506024096385543, "em_norm_relaxed_stderr": 0.010750305246379226, "acc": 0.3140562248995984, "acc_stderr": 0.010735367420948657 }, "driving_test_SK_AVERAGE|0": { "em_relaxed": 0.2710413694721826, "acc": 0.2710413694721826, "em_relaxed_stderr": 0.016450865339756114, "acc_stderr": 0.016450865339756114, "em_relaxed__prompts": { "3": 0.17261055634807418, "1": 0.3666191155492154, "2": 0.2738944365192582 }, "acc__prompts": { "3": 0.17261055634807418, "1": 0.3666191155492154, "2": 0.2738944365192582 } }, "driving_test_AVERAGE|0": { "em_relaxed": 0.2710413694721826, "em_relaxed_stderr": 0.016450865339756114, "acc": 0.2710413694721826, "acc_stderr": 0.016450865339756114 }, "sklep_sentiment_analysis_SK_AVERAGE|0": { "em_norm_relaxed": 0.11388355726167626, "acc": 0.10748560460652591, "em_norm_relaxed_stderr": 0.009453015726623323, "acc_stderr": 0.009172256989852929, "em_norm_relaxed__prompts": { "2": 0.19193857965451055, "3": 0.09404990403071017, "1": 0.05566218809980806 }, "acc__prompts": { "2": 0.1833013435700576, "3": 0.09213051823416507, "1": 0.04702495201535509 } }, "sklep_sentiment_analysis_AVERAGE|0": { "em_norm_relaxed": 0.11388355726167626, "em_norm_relaxed_stderr": 0.009453015726623323, "acc": 0.10748560460652591, "acc_stderr": 0.009172256989852929 }, "slovakcopa_SK_AVERAGE|0": { "em_relaxed": 0.27066666666666667, "acc": 0.27, "em_relaxed_stderr": 0.016480088564366616, "acc_stderr": 0.016473550449007283, "em_relaxed__prompts": { "1": 0.402, "2": 0.394, "3": 0.016 }, "acc__prompts": { "1": 0.402, "2": 0.392, "3": 0.016 } }, "slovakcopa_AVERAGE|0": { "em_relaxed": 0.27066666666666667, "em_relaxed_stderr": 0.016480088564366616, "acc": 0.27, "acc_stderr": 0.016473550449007283 }, "sklep_qa_SK_AVERAGE|0": { "qf1": 0.14283353603077906, "qem": 0.05203659257713312, "qf1_stderr": 0.0027203043184777205, "qem_stderr": 0.002268897741053689, "qf1__prompts": { "2": 0.14428713100967, "1": 0.1424834912487429, "3": 0.1417299858339243 }, "qem__prompts": { "2": 0.05238443076280914, "1": 0.05248878221851195, "3": 0.05123656475007826 } }, "sklep_qa_AVERAGE|0": { "qf1": 0.14283353603077906, "qf1_stderr": 0.0027203043184777205, "qem": 0.05203659257713312, "qem_stderr": 0.002268897741053689 }, "senti_sk_3class_SK_AVERAGE|0": { "em_norm_relaxed": 0.38621623547858314, "acc": 0.3813698239612287, "em_norm_relaxed_stderr": 0.006810719788072698, "acc_stderr": 0.006801058501709849, "em_norm_relaxed__prompts": { "3": 0.5809279452640581, "2": 0.32050459696386574, "1": 0.2572161642078255 }, "acc__prompts": { "3": 0.573444515715202, "2": 0.3158007269617276, "1": 0.25486422920675644 } }, "senti_sk_3class_AVERAGE|0": { "em_norm_relaxed": 0.38621623547858314, "em_norm_relaxed_stderr": 0.006810719788072698, "acc": 0.3813698239612287, "acc_stderr": 0.006801058501709849 }, "multi_wiki_qa_SK_AVERAGE|0": { "qf1": 0.12354806704071784, "qem": 0.07884730838503101, "qf1_stderr": 0.003950374051156623, "qem_stderr": 0.0038122173887514235, "qf1__prompts": { "3": 0.12171966052606027, "2": 0.12697709233547141, "1": 0.12194744826062183 }, "qem__prompts": { "3": 0.07884730838503101, "2": 0.08204922953772263, "1": 0.0756453872323394 } }, "multi_wiki_qa_AVERAGE|0": { "qf1": 0.12354806704071784, "qf1_stderr": 0.003950374051156623, "qem": 0.07884730838503101, "qem_stderr": 0.0038122173887514235 }, "ifeval_sk_SK_AVERAGE|0": { "prompt_level_strict_acc": 0.10242085661080075, "prompt_level_loose_acc": 0.10242085661080075, "inst_level_strict_acc": 0.21307506053268765, "inst_level_loose_acc": 0.21307506053268765, "prompt_level_strict_acc_stderr": 0.01309628698247887, "prompt_level_loose_acc_stderr": 0.01309628698247887, "inst_level_strict_acc_stderr": 0.00041218013349389904, "inst_level_loose_acc_stderr": 0.00041218013349389904 }, "ifeval_sk_AVERAGE|0": { "prompt_level_strict_acc": 0.10242085661080075, "prompt_level_strict_acc_stderr": 0.01309628698247887, "prompt_level_loose_acc": 0.10242085661080075, "prompt_level_loose_acc_stderr": 0.01309628698247887, "inst_level_strict_acc": 0.21307506053268765, "inst_level_strict_acc_stderr": 0.00041218013349389904, "inst_level_loose_acc": 0.21307506053268765, "inst_level_loose_acc_stderr": 0.00041218013349389904 }, "globalmmlu_lite_SK_AVERAGE|0": { "em_relaxed": 0.2783333333333333, "acc": 0.2758333333333333, "em_relaxed_stderr": 0.022429986391836288, "acc_stderr": 0.022369608398602674, "em_relaxed__prompts": { "3": 0.2925, "1": 0.2725, "2": 0.27 }, "acc__prompts": { "3": 0.2875, "1": 0.2725, "2": 0.2675 } }, "globalmmlu_lite_AVERAGE|0": { "em_relaxed": 0.2783333333333333, "em_relaxed_stderr": 0.022429986391836288, "acc": 0.2758333333333333, "acc_stderr": 0.022369608398602674 }, "sklep_nli_SK_AVERAGE|0": { "em_norm_relaxed": 0.024047428723687717, "acc": 0.0187183586464162, "em_norm_relaxed_stderr": 0.0017224579545798097, "acc_stderr": 0.001563138066747669, "em_norm_relaxed__prompts": { "3": 0.002398081534772182, "1": 0.004996003197442046, "2": 0.06474820143884892 }, "acc__prompts": { "3": 0.002398081534772182, "1": 0.004396482813749001, "2": 0.04936051159072742 } }, "sklep_nli_AVERAGE|0": { "em_norm_relaxed": 0.024047428723687717, "em_norm_relaxed_stderr": 0.0017224579545798097, "acc": 0.0187183586464162, "acc_stderr": 0.001563138066747669 }, "slovak_tests_qa_SK_AVERAGE|0": { "rouge2": 0.0030834556044640076, "rouge1": 0.0149009568631566, "rougeL": 0.0149009568631566, "qf1": 0.008916490572837322, "qem": 0.00980392156862745, "rouge2_stderr": 0.00228223362995418, "rouge1_stderr": 0.00523977237139308, "rougeL_stderr": 0.00523977237139308, "qf1_stderr": 0.005125765723300312, "qem_stderr": 0.006367605535927983, "rouge2__prompts": { "1": 0.003274643190609577, "2": 0.0021008403361344537, "3": 0.0038748832866479918 }, "rouge1__prompts": { "1": 0.014745243264740951, "2": 0.015252537196116269, "3": 0.014705090128612585 }, "rougeL__prompts": { "1": 0.014745243264740951, "2": 0.015252537196116269, "3": 0.014705090128612585 }, "qf1__prompts": { "1": 0.00765639589169001, "2": 0.006349206349206349, "3": 0.012743869477615606 }, "qem__prompts": { "1": 0.008403361344537815, "2": 0.008403361344537815, "3": 0.012605042016806723 } }, "slovak_tests_qa_AVERAGE|0": { "rouge2": 0.0030834556044640076, "rouge2_stderr": 0.00228223362995418, "rouge1": 0.0149009568631566, "rouge1_stderr": 0.00523977237139308, "rougeL": 0.0149009568631566, "rougeL_stderr": 0.00523977237139308, "qf1": 0.008916490572837322, "qf1_stderr": 0.005125765723300312, "qem": 0.00980392156862745, "qem_stderr": 0.006367605535927983 }, "skj1_SK_AVERAGE|0": { "em_relaxed": 0.0033333333333333335, "acc": 0.0, "em_relaxed_stderr": 0.0033333333333333322, "acc_stderr": 0.0, "em_relaxed__prompts": { "1": 0.01, "3": 0.0, "2": 0.0 }, "acc__prompts": { "1": 0.0, "3": 0.0, "2": 0.0 } }, "skj1_AVERAGE|0": { "em_relaxed": 0.0033333333333333335, "em_relaxed_stderr": 0.0033333333333333322, "acc": 0.0, "acc_stderr": 0.0 }, "financial_exams_SK_AVERAGE|0": { "em_relaxed": 0.18640679660169915, "acc": 0.18615692153923039, "em_relaxed_stderr": 0.010665889604449459, "acc_stderr": 0.010660482281487506, "em_relaxed__prompts": { "3": 0.18590704647676162, "1": 0.18290854572713644, "2": 0.1904047976011994 }, "acc__prompts": { "3": 0.18590704647676162, "1": 0.18290854572713644, "2": 0.1896551724137931 } }, "financial_exams_AVERAGE|0": { "em_relaxed": 0.18640679660169915, "em_relaxed_stderr": 0.010665889604449459, "acc": 0.18615692153923039, "acc_stderr": 0.010660482281487506 }, "truthful_qa_SK_AVERAGE|0": { "em_relaxed": 0.32390745501285345, "acc": 0.31876606683804626, "em_relaxed_stderr": 0.01598799102785622, "acc_stderr": 0.015861196644299858, "em_relaxed__prompts": { "2": 0.40359897172236503, "3": 0.14524421593830333, "1": 0.42287917737789205 }, "acc__prompts": { "2": 0.397172236503856, "3": 0.13624678663239073, "1": 0.42287917737789205 } }, "truthful_qa_AVERAGE|0": { "em_relaxed": 0.32390745501285345, "em_relaxed_stderr": 0.01598799102785622, "acc": 0.31876606683804626, "acc_stderr": 0.015861196644299858 }, "sklep_hate_speech_SK_AVERAGE|0": { "em_norm_relaxed": 0.21430376547889815, "acc": 0.2137983320697498, "em_norm_relaxed_stderr": 0.010710417940514876, "acc_stderr": 0.010698565134938456, "em_norm_relaxed__prompts": { "3": 0.09401061410159212, "1": 0.35329795299469297, "2": 0.1956027293404094 }, "acc__prompts": { "3": 0.0932524639878696, "1": 0.3525398028809704, "2": 0.1956027293404094 } }, "sklep_hate_speech_AVERAGE|0": { "em_norm_relaxed": 0.21430376547889815, "em_norm_relaxed_stderr": 0.010710417940514876, "acc": 0.2137983320697498, "acc_stderr": 0.010698565134938456 }, "toxic_sk_SK_AVERAGE|0": { "em_norm_relaxed": 0.19977298524404086, "acc": 0.19901626939084374, "em_norm_relaxed_stderr": 0.011151493876554907, "acc_stderr": 0.011139685234547891, "em_norm_relaxed__prompts": { "3": 0.007945516458569807, "2": 0.23723041997729852, "1": 0.35414301929625425 }, "acc__prompts": { "3": 0.007945516458569807, "2": 0.23609534619750283, "1": 0.3530079455164586 } }, "toxic_sk_AVERAGE|0": { "em_norm_relaxed": 0.19977298524404086, "em_norm_relaxed_stderr": 0.011151493876554907, "acc": 0.19901626939084374, "acc_stderr": 0.011139685234547891 }, "slohry_SK_AVERAGE|0": { "em_norm_relaxed": 0.016983016983016984, "acc": 0.016983016983016984, "em_norm_relaxed_stderr": 0.004436902963904142, "acc_stderr": 0.004436902963904142, "em_norm_relaxed__prompts": { "1": 0.018731268731268732, "3": 0.016983016983016984, "2": 0.015234765234765234 }, "acc__prompts": { "1": 0.018731268731268732, "3": 0.016983016983016984, "2": 0.015234765234765234 } }, "slohry_AVERAGE|0": { "em_norm_relaxed": 0.016983016983016984, "em_norm_relaxed_stderr": 0.004436902963904142, "acc": 0.016983016983016984, "acc_stderr": 0.004436902963904142 }, "chiby_SK_AVERAGE|0": { "em_norm_relaxed": 0.14238539238539238, "acc": 0.1411227661227661, "em_norm_relaxed_stderr": 0.005425997886686588, "acc_stderr": 0.005408227359113877, "em_norm_relaxed__prompts": { "2": 0.02564102564102564, "3": 0.16258741258741258, "1": 0.23892773892773891 }, "acc__prompts": { "2": 0.025349650349650348, "3": 0.1622960372960373, "1": 0.23572261072261072 } }, "chiby_AVERAGE|0": { "em_norm_relaxed": 0.14238539238539238, "em_norm_relaxed_stderr": 0.005425997886686588, "acc": 0.1411227661227661, "acc_stderr": 0.005408227359113877 }, "slovak_trivia_SK_AVERAGE|0": { "em_relaxed": 0.2785646836638338, "acc": 0.2785646836638338, "em_relaxed_stderr": 0.023888375986348194, "acc_stderr": 0.023888375986348194, "em_relaxed__prompts": { "3": 0.28611898016997167, "1": 0.26628895184135976, "2": 0.28328611898017 }, "acc__prompts": { "3": 0.28611898016997167, "1": 0.26628895184135976, "2": 0.28328611898017 } }, "slovak_trivia_AVERAGE|0": { "em_relaxed": 0.2785646836638338, "em_relaxed_stderr": 0.023888375986348194, "acc": 0.2785646836638338, "acc_stderr": 0.023888375986348194 }, "analogies_sk_SK_AVERAGE|0": { "acc": 0.000628693574751666, "acc_stderr": 0.00039549148554889166, "acc__prompts": { "3": 0.0, "2": 0.0007544322897019992, "1": 0.001131648434552999 } }, "analogies_sk_AVERAGE|0": { "acc": 0.000628693574751666, "acc_stderr": 0.00039549148554889166 }, "SK_AVERAGE|0": { "em_norm_relaxed": 0.20551192367489274, "acc": 0.21129673365203147, "em_relaxed": 0.24052159099864373, "rouge2": 0.00315582529372049, "rouge1": 0.014859863107823173, "rougeL": 0.014859863107823173, "qf1": 0.07079407895918113, "qem": 0.038181737423518054, "prompt_level_strict_acc": 0.10242085661080075, "prompt_level_loose_acc": 0.10242085661080075, "inst_level_strict_acc": 0.21307506053268765, "inst_level_loose_acc": 0.21307506053268765, "em_norm_relaxed_stderr": 0.0025722619842950684, "acc_stderr": 0.0031822329617778393, "em_relaxed_stderr": 0.0056971140284625835, "rouge2_stderr": 0.001710070063230727, "rouge1_stderr": 0.0047917616644827035, "rougeL_stderr": 0.0047917616644827035, "qf1_stderr": 0.002234105218066023, "qem_stderr": 0.0029234626947495608, "prompt_level_strict_acc_stderr": 0.01309628698247887, "prompt_level_loose_acc_stderr": 0.01309628698247887, "inst_level_strict_acc_stderr": 0.00041218013349389904, "inst_level_loose_acc_stderr": 0.00041218013349389904 }, "AVERAGE|0": { "em_norm_relaxed": 0.185433666807816, "acc": 0.21317053562216026, "em_relaxed": 0.2602872791374196, "rouge2": 0.00315582529372049, "rouge1": 0.014859863107823173, "rougeL": 0.014859863107823173, "qf1": 0.07079407895918113, "qem": 0.038181737423518054, "prompt_level_strict_acc": 0.10242085661080075, "prompt_level_loose_acc": 0.10242085661080075, "inst_level_strict_acc": 0.21307506053268765, "inst_level_loose_acc": 0.21307506053268765, "em_norm_relaxed_stderr": 0.002756064620435493, "acc_stderr": 0.003457735075973522, "em_relaxed_stderr": 0.0061656193838727944, "rouge2_stderr": 0.001710070063230727, "rouge1_stderr": 0.0047917616644827035, "rougeL_stderr": 0.0047917616644827035, "qf1_stderr": 0.002234105218066023, "qem_stderr": 0.0029234626947495608, "prompt_level_strict_acc_stderr": 0.01309628698247887, "prompt_level_loose_acc_stderr": 0.01309628698247887, "inst_level_strict_acc_stderr": 0.00041218013349389904, "inst_level_loose_acc_stderr": 0.00041218013349389904 } } }