3250 lines
112 KiB
JSON
3250 lines
112 KiB
JSON
{
|
|
"model_name": "models/ministral-last-109064",
|
|
"config_general": {
|
|
"lighteval_sha": "?",
|
|
"num_fewshot_seeds": 1,
|
|
"max_samples": null,
|
|
"job_id": "0",
|
|
"start_time": 883355.295419078,
|
|
"end_time": 883503.394399547,
|
|
"total_evaluation_time_secondes": "148.0989804690471",
|
|
"model_config": {
|
|
"model_name": "models/ministral-last-109064",
|
|
"generation_parameters": {
|
|
"num_blocks": null,
|
|
"block_size": null,
|
|
"early_stopping": null,
|
|
"repetition_penalty": null,
|
|
"frequency_penalty": null,
|
|
"length_penalty": null,
|
|
"presence_penalty": null,
|
|
"max_new_tokens": null,
|
|
"min_new_tokens": null,
|
|
"seed": null,
|
|
"stop_tokens": null,
|
|
"temperature": 0,
|
|
"top_k": null,
|
|
"min_p": null,
|
|
"top_p": null,
|
|
"truncate_prompt": null,
|
|
"cache_implementation": null,
|
|
"response_format": null,
|
|
"skip_special_tokens": true
|
|
},
|
|
"system_prompt": null,
|
|
"cache_dir": "~/.cache/huggingface/lighteval",
|
|
"tokenizer": null,
|
|
"revision": "main",
|
|
"dtype": "bfloat16",
|
|
"tensor_parallel_size": 1,
|
|
"data_parallel_size": 1,
|
|
"pipeline_parallel_size": 1,
|
|
"gpu_memory_utilization": 0.9,
|
|
"enable_prefix_caching": null,
|
|
"max_model_length": null,
|
|
"quantization": null,
|
|
"load_format": null,
|
|
"swap_space": 4,
|
|
"seed": 1234,
|
|
"trust_remote_code": true,
|
|
"add_special_tokens": true,
|
|
"multichoice_continuations_start_space": true,
|
|
"pairwise_tokenization": false,
|
|
"max_num_seqs": 128,
|
|
"max_num_batched_tokens": 2048,
|
|
"subfolder": null,
|
|
"is_async": false,
|
|
"override_chat_template": null,
|
|
"enforce_eager": false
|
|
},
|
|
"model_name": "models/ministral-last-109064"
|
|
},
|
|
"versions": {},
|
|
"results": {
|
|
"movie_review.prompt1_sentiment_binary_sk|0": {
|
|
"acc": 0.09860557768924302,
|
|
"acc_stderr": 0.006654792030522998,
|
|
"em_norm_relaxed": 0.10109561752988047,
|
|
"em_norm_relaxed_stderr": 0.006728979843112432
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": {
|
|
"acc": 0.1,
|
|
"acc_stderr": 0.06882472016116853
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"acc": 0.21983914209115282,
|
|
"acc_stderr": 0.021472030675572816,
|
|
"em_relaxed": 0.21983914209115282,
|
|
"em_relaxed_stderr": 0.021472030675572816
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
|
|
"acc": 0.27350427350427353,
|
|
"acc_stderr": 0.029202540153431183,
|
|
"em_relaxed": 0.27350427350427353,
|
|
"em_relaxed_stderr": 0.029202540153431183
|
|
},
|
|
"belebele.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.27111111111111114,
|
|
"acc_stderr": 0.014826016446581967,
|
|
"em_relaxed": 0.2722222222222222,
|
|
"em_relaxed_stderr": 0.014845038794433947
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.2318840579710145,
|
|
"acc_stderr": 0.05117930441535768,
|
|
"em_relaxed": 0.2318840579710145,
|
|
"em_relaxed_stderr": 0.05117930441535768
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
|
|
"acc": 0.254601226993865,
|
|
"acc_stderr": 0.024164783161712056,
|
|
"em_relaxed": 0.254601226993865,
|
|
"em_relaxed_stderr": 0.024164783161712056
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
|
|
"acc": 0.20202020202020202,
|
|
"acc_stderr": 0.0405583530232473,
|
|
"em_relaxed": 0.2222222222222222,
|
|
"em_relaxed_stderr": 0.041996052556580815
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"acc": 0.20522388059701493,
|
|
"acc_stderr": 0.017460611985170203,
|
|
"em_relaxed": 0.20522388059701493,
|
|
"em_relaxed_stderr": 0.017460611985170203
|
|
},
|
|
"senti_sk.prompt2_sentiment_binary_sk|0": {
|
|
"acc": 0.32135193133047213,
|
|
"acc_stderr": 0.007649500362664599,
|
|
"em_norm_relaxed": 0.3261802575107296,
|
|
"em_norm_relaxed_stderr": 0.007679288875174346
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": {
|
|
"acc": 0.18095238095238095,
|
|
"acc_stderr": 0.03775026958386237,
|
|
"em_relaxed": 0.18095238095238095,
|
|
"em_relaxed_stderr": 0.03775026958386237
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": {
|
|
"acc": 0.05357142857142857,
|
|
"acc_stderr": 0.030361917118846844
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
|
|
"acc": 0.27181208053691275,
|
|
"acc_stderr": 0.018238857466939602,
|
|
"em_relaxed": 0.27181208053691275,
|
|
"em_relaxed_stderr": 0.018238857466939602
|
|
},
|
|
"e_tests.prompt3_qa_sk:chemia|0": {
|
|
"qem": 0.037037037037037035,
|
|
"qem_stderr": 0.025940896147128693,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.02444444444444445,
|
|
"rouge1_stderr": 0.011223747742795042,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.02444444444444445,
|
|
"rougeL_stderr": 0.011223747742795042
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:vsp|0": {
|
|
"acc": 0.2657766990291262,
|
|
"acc_stderr": 0.015398294286338463,
|
|
"em_relaxed": 0.27184466019417475,
|
|
"em_relaxed_stderr": 0.015508596797488939
|
|
},
|
|
"arc.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.24123182207014543,
|
|
"acc_stderr": 0.01251844632394384,
|
|
"em_relaxed": 0.24721984602224123,
|
|
"em_relaxed_stderr": 0.012622759999132595
|
|
},
|
|
"sklep.prompt3_nli_binary_sk:rte|0": {
|
|
"acc": 0.1650602409638554,
|
|
"acc_stderr": 0.009114348186638069,
|
|
"em_norm_relaxed": 0.16566265060240964,
|
|
"em_norm_relaxed_stderr": 0.009127670467992892
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
|
|
"acc": 0.1513157894736842,
|
|
"acc_stderr": 0.029162631596843982,
|
|
"em_relaxed": 0.1513157894736842,
|
|
"em_relaxed_stderr": 0.029162631596843982
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
|
|
"acc": 0.25892857142857145,
|
|
"acc_stderr": 0.041577515398656284,
|
|
"em_relaxed": 0.2767857142857143,
|
|
"em_relaxed_stderr": 0.04246624336697624
|
|
},
|
|
"driving_test.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.17261055634807418,
|
|
"acc_stderr": 0.014283664605079543,
|
|
"em_relaxed": 0.17261055634807418,
|
|
"em_relaxed_stderr": 0.014283664605079543
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"acc": 0.23232323232323232,
|
|
"acc_stderr": 0.04266016017054689,
|
|
"em_relaxed": 0.23232323232323232,
|
|
"em_relaxed_stderr": 0.04266016017054689
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
|
|
"acc": 0.24679760888129804,
|
|
"acc_stderr": 0.01260472537669998,
|
|
"em_relaxed": 0.2493595217762596,
|
|
"em_relaxed_stderr": 0.012648412983648835
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
|
|
"acc": 0.25217391304347825,
|
|
"acc_stderr": 0.040672227541547167,
|
|
"em_relaxed": 0.2608695652173913,
|
|
"em_relaxed_stderr": 0.04112631751856164
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"acc": 0.2676767676767677,
|
|
"acc_stderr": 0.03154449888270286,
|
|
"em_relaxed": 0.2676767676767677,
|
|
"em_relaxed_stderr": 0.03154449888270286
|
|
},
|
|
"senti_sk.prompt1_sentiment_binary_sk|0": {
|
|
"acc": 0.39914163090128757,
|
|
"acc_stderr": 0.0080217638856134,
|
|
"em_norm_relaxed": 0.4069206008583691,
|
|
"em_norm_relaxed_stderr": 0.008046954795106477
|
|
},
|
|
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"acc": 0.1833013435700576,
|
|
"acc_stderr": 0.011991910538467281,
|
|
"em_norm_relaxed": 0.19193857965451055,
|
|
"em_norm_relaxed_stderr": 0.012206129123492155
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
|
|
"acc": 0.2222222222222222,
|
|
"acc_stderr": 0.03188578017686398,
|
|
"em_relaxed": 0.2222222222222222,
|
|
"em_relaxed_stderr": 0.03188578017686398
|
|
},
|
|
"e_tests.prompt2_qa_sk:fyzika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.014285714285714287,
|
|
"rouge1_stderr": 0.014285714285714287,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.014285714285714287,
|
|
"rougeL_stderr": 0.014285714285714287
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:che|0": {
|
|
"acc": 0.2601626016260163,
|
|
"acc_stderr": 0.028028995361669362,
|
|
"em_relaxed": 0.2601626016260163,
|
|
"em_relaxed_stderr": 0.028028995361669362
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
|
|
"acc": 0.17,
|
|
"acc_stderr": 0.0377525168068637,
|
|
"em_relaxed": 0.17,
|
|
"em_relaxed_stderr": 0.0377525168068637
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
|
|
"acc": 0.2408198121263877,
|
|
"acc_stderr": 0.012500448888138267,
|
|
"em_relaxed": 0.2408198121263877,
|
|
"em_relaxed_stderr": 0.012500448888138267
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
|
|
"acc": 0.25153374233128833,
|
|
"acc_stderr": 0.024068141649530538,
|
|
"em_relaxed": 0.25153374233128833,
|
|
"em_relaxed_stderr": 0.024068141649530538
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
|
|
"acc": 0.22826086956521738,
|
|
"acc_stderr": 0.025309563050037266,
|
|
"em_relaxed": 0.22826086956521738,
|
|
"em_relaxed_stderr": 0.025309563050037266
|
|
},
|
|
"e_tests.prompt2_qa_sk:geografia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.01904761904761905,
|
|
"qf1_stderr": 0.019047619047619056,
|
|
"rouge1": 0.01360544217687075,
|
|
"rouge1_stderr": 0.013605442176870748,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.01360544217687075,
|
|
"rougeL_stderr": 0.013605442176870748
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"acc": 0.29292929292929293,
|
|
"acc_stderr": 0.045972676254181784,
|
|
"em_relaxed": 0.29292929292929293,
|
|
"em_relaxed_stderr": 0.045972676254181784
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": {
|
|
"acc": 0.05,
|
|
"acc_stderr": 0.04999999999999998
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:fyzika|0": {
|
|
"acc": 0.13636363636363635,
|
|
"acc_stderr": 0.07488677009526491,
|
|
"em_relaxed": 0.13636363636363635,
|
|
"em_relaxed_stderr": 0.07488677009526491
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"acc": 0.24067164179104478,
|
|
"acc_stderr": 0.018482060138670608,
|
|
"em_relaxed": 0.24253731343283583,
|
|
"em_relaxed_stderr": 0.01853075065058218
|
|
},
|
|
"slovakcopa.prompt1_copa_sk|0": {
|
|
"acc": 0.402,
|
|
"acc_stderr": 0.021948929609938612,
|
|
"em_relaxed": 0.402,
|
|
"em_relaxed_stderr": 0.021948929609938612
|
|
},
|
|
"movie_review.prompt3_sentiment_binary_sk|0": {
|
|
"acc": 0.20766932270916336,
|
|
"acc_stderr": 0.009054526051113992,
|
|
"em_norm_relaxed": 0.20916334661354583,
|
|
"em_norm_relaxed_stderr": 0.009078466587423147
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
|
|
"acc": 0.20348837209302326,
|
|
"acc_stderr": 0.03078703062133896,
|
|
"em_relaxed": 0.20348837209302326,
|
|
"em_relaxed_stderr": 0.03078703062133896
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": {
|
|
"acc": 0.034482758620689655,
|
|
"acc_stderr": 0.034482758620689676
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:geografia|0": {
|
|
"acc": 0.20408163265306123,
|
|
"acc_stderr": 0.05817221556628253,
|
|
"em_relaxed": 0.20408163265306123,
|
|
"em_relaxed_stderr": 0.05817221556628253
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"acc": 0.22932330827067668,
|
|
"acc_stderr": 0.02582481829225878,
|
|
"em_relaxed": 0.22932330827067668,
|
|
"em_relaxed_stderr": 0.02582481829225878
|
|
},
|
|
"sklep.prompt2_qa_sk:question-answering|0": {
|
|
"qem": 0.05238443076280914,
|
|
"qem_stderr": 0.0022760883977866997,
|
|
"qf1": 0.14428713100967,
|
|
"qf1_stderr": 0.002729924096699559
|
|
},
|
|
"e_tests.prompt2_qa_sk:informatika|0": {
|
|
"qem": 0.038461538461538464,
|
|
"qem_stderr": 0.021915495473007307,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0036324786324786326,
|
|
"rouge1_stderr": 0.0025563439430967027,
|
|
"rouge2": 0.0008547008547008547,
|
|
"rouge2_stderr": 0.0008547008547008546,
|
|
"rougeL": 0.0036324786324786326,
|
|
"rougeL_stderr": 0.0025563439430967027
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"acc": 0.32340425531914896,
|
|
"acc_stderr": 0.030579442773610337,
|
|
"em_relaxed": 0.32340425531914896,
|
|
"em_relaxed_stderr": 0.030579442773610337
|
|
},
|
|
"e_tests.prompt1_qa_sk:dejepis|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"acc": 0.2482758620689655,
|
|
"acc_stderr": 0.03600105692727771,
|
|
"em_relaxed": 0.2482758620689655,
|
|
"em_relaxed_stderr": 0.03600105692727771
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
|
|
"acc": 0.3926940639269406,
|
|
"acc_stderr": 0.03307521564995785,
|
|
"em_relaxed": 0.3926940639269406,
|
|
"em_relaxed_stderr": 0.03307521564995785
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
|
|
"acc": 0.31,
|
|
"acc_stderr": 0.04648231987117316,
|
|
"em_relaxed": 0.31,
|
|
"em_relaxed_stderr": 0.04648231987117316
|
|
},
|
|
"e_tests.prompt1_qa_sk:obcianska-nauka|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.020408163265306124,
|
|
"rouge1_stderr": 0.020408163265306124,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.020408163265306124,
|
|
"rougeL_stderr": 0.020408163265306124
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"acc": 0.2548262548262548,
|
|
"acc_stderr": 0.027129450992473495,
|
|
"em_relaxed": 0.2625482625482625,
|
|
"em_relaxed_stderr": 0.027394382974098772
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"acc": 0.19626168224299065,
|
|
"acc_stderr": 0.027213555922450262,
|
|
"em_relaxed": 0.205607476635514,
|
|
"em_relaxed_stderr": 0.027691547344010747
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
|
|
"acc": 0.23478260869565218,
|
|
"acc_stderr": 0.039698395317531235,
|
|
"em_relaxed": 0.23478260869565218,
|
|
"em_relaxed_stderr": 0.039698395317531235
|
|
},
|
|
"senti_sk.prompt3_sentiment_3_class_sk|0": {
|
|
"acc": 0.573444515715202,
|
|
"acc_stderr": 0.007232629453324713,
|
|
"em_norm_relaxed": 0.5809279452640581,
|
|
"em_norm_relaxed_stderr": 0.007215529950821413
|
|
},
|
|
"e_tests.prompt1_qa_sk:biologia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.019047619047619046,
|
|
"qf1_stderr": 0.019047619047619035,
|
|
"rouge1": 0.04,
|
|
"rouge1_stderr": 0.022529159443043678,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.04,
|
|
"rougeL_stderr": 0.022529159443043678
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": {
|
|
"acc": 0.27371273712737126,
|
|
"acc_stderr": 0.023242230741504617,
|
|
"em_relaxed": 0.27371273712737126,
|
|
"em_relaxed_stderr": 0.023242230741504617
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
|
|
"acc": 0.35365853658536583,
|
|
"acc_stderr": 0.03744805613781578,
|
|
"em_relaxed": 0.35365853658536583,
|
|
"em_relaxed_stderr": 0.03744805613781578
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
|
|
"acc": 0.23857868020304568,
|
|
"acc_stderr": 0.03044390069569734,
|
|
"em_relaxed": 0.23857868020304568,
|
|
"em_relaxed_stderr": 0.03044390069569734
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
|
|
"acc": 0.3926940639269406,
|
|
"acc_stderr": 0.03307521564995785,
|
|
"em_relaxed": 0.3926940639269406,
|
|
"em_relaxed_stderr": 0.03307521564995785
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": {
|
|
"acc": 0.022727272727272728,
|
|
"acc_stderr": 0.022727272727272728
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
|
|
"acc": 0.2926829268292683,
|
|
"acc_stderr": 0.035637888362588285,
|
|
"em_relaxed": 0.2926829268292683,
|
|
"em_relaxed_stderr": 0.035637888362588285
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
|
|
"acc": 0.2323943661971831,
|
|
"acc_stderr": 0.03556905428023102,
|
|
"em_relaxed": 0.23943661971830985,
|
|
"em_relaxed_stderr": 0.03593796120702438
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"acc": 0.2534562211981567,
|
|
"acc_stderr": 0.02959731008909016,
|
|
"em_relaxed": 0.2534562211981567,
|
|
"em_relaxed_stderr": 0.02959731008909016
|
|
},
|
|
"multi_wiki_qa.prompt3_qa_sk|0": {
|
|
"qem": 0.07884730838503101,
|
|
"qem_stderr": 0.0038128351969214953,
|
|
"qf1": 0.12171966052606027,
|
|
"qf1_stderr": 0.003955037338259647
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.02857142857142857,
|
|
"acc_stderr": 0.014130710990583526
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:biologia|0": {
|
|
"acc": 0.3103448275862069,
|
|
"acc_stderr": 0.049887188500387446,
|
|
"em_relaxed": 0.3103448275862069,
|
|
"em_relaxed_stderr": 0.049887188500387446
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
|
|
"acc": 0.24,
|
|
"acc_stderr": 0.04292346959909284,
|
|
"em_relaxed": 0.24,
|
|
"em_relaxed_stderr": 0.04292346959909284
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": {
|
|
"acc": 0.034482758620689655,
|
|
"acc_stderr": 0.034482758620689676
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"acc": 0.29,
|
|
"acc_stderr": 0.045604802157206845,
|
|
"em_relaxed": 0.29,
|
|
"em_relaxed_stderr": 0.045604802157206845
|
|
},
|
|
"ifeval_sk.ifeval|0": {
|
|
"inst_level_loose_acc": 0.21307506053268765,
|
|
"inst_level_loose_acc_stderr": 0.00041218013349389904,
|
|
"inst_level_strict_acc": 0.21307506053268765,
|
|
"inst_level_strict_acc_stderr": 0.00041218013349389904,
|
|
"prompt_level_loose_acc": 0.10242085661080075,
|
|
"prompt_level_loose_acc_stderr": 0.01309628698247887,
|
|
"prompt_level_strict_acc": 0.10242085661080075,
|
|
"prompt_level_strict_acc_stderr": 0.01309628698247887
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
|
|
"acc": 0.18181818181818182,
|
|
"acc_stderr": 0.027479603010538804,
|
|
"em_relaxed": 0.18181818181818182,
|
|
"em_relaxed_stderr": 0.027479603010538804
|
|
},
|
|
"e_tests.prompt1_qa_sk:geografia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"acc": 0.22429906542056074,
|
|
"acc_stderr": 0.015251413306651283,
|
|
"em_relaxed": 0.22429906542056074,
|
|
"em_relaxed_stderr": 0.015251413306651283
|
|
},
|
|
"e_tests.prompt3_qa_sk:slovensky-jazyk|0": {
|
|
"qem": 0.0051813471502590676,
|
|
"qem_stderr": 0.00518134715025908,
|
|
"qf1": 0.008737173625927054,
|
|
"qf1_stderr": 0.005740091786106941,
|
|
"rouge1": 0.016785287251608494,
|
|
"rouge1_stderr": 0.006781696658405232,
|
|
"rouge2": 0.00659444182760245,
|
|
"rouge2_stderr": 0.005363480846922639,
|
|
"rougeL": 0.016785287251608494,
|
|
"rougeL_stderr": 0.006781696658405232
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
|
|
"acc": 0.26495726495726496,
|
|
"acc_stderr": 0.028911208802749472,
|
|
"em_relaxed": 0.26495726495726496,
|
|
"em_relaxed_stderr": 0.028911208802749472
|
|
},
|
|
"driving_test.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.3666191155492154,
|
|
"acc_stderr": 0.01821339929295096,
|
|
"em_relaxed": 0.3666191155492154,
|
|
"em_relaxed_stderr": 0.01821339929295096
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:aj|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.13333333333333333,
|
|
"em_relaxed": 0.2,
|
|
"em_relaxed_stderr": 0.13333333333333333
|
|
},
|
|
"sklep.prompt2_nli_binary_sk:rte|0": {
|
|
"acc": 0.5132530120481927,
|
|
"acc_stderr": 0.012271402375351256,
|
|
"em_norm_relaxed": 0.5132530120481927,
|
|
"em_norm_relaxed_stderr": 0.012271402375351256
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
|
|
"acc": 0.23857868020304568,
|
|
"acc_stderr": 0.03044390069569734,
|
|
"em_relaxed": 0.23857868020304568,
|
|
"em_relaxed_stderr": 0.03044390069569734
|
|
},
|
|
"multi_wiki_qa.prompt2_qa_sk|0": {
|
|
"qem": 0.08204922953772263,
|
|
"qem_stderr": 0.003882716972284686,
|
|
"qf1": 0.12697709233547141,
|
|
"qf1_stderr": 0.0040052643032774445
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:geografia|0": {
|
|
"acc": 0.22448979591836735,
|
|
"acc_stderr": 0.06022425581505364,
|
|
"em_relaxed": 0.22448979591836735,
|
|
"em_relaxed_stderr": 0.06022425581505364
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
|
|
"acc": 0.26,
|
|
"acc_stderr": 0.0440844002276808,
|
|
"em_relaxed": 0.26,
|
|
"em_relaxed_stderr": 0.0440844002276808
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
|
|
"acc": 0.22442244224422442,
|
|
"acc_stderr": 0.024007242951044422,
|
|
"em_relaxed": 0.2376237623762376,
|
|
"em_relaxed_stderr": 0.0244921064409582
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.04020151261036845,
|
|
"em_relaxed": 0.2,
|
|
"em_relaxed_stderr": 0.04020151261036845
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"acc": 0.21705426356589147,
|
|
"acc_stderr": 0.020982456910832083,
|
|
"em_relaxed": 0.21705426356589147,
|
|
"em_relaxed_stderr": 0.020982456910832083
|
|
},
|
|
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.2875,
|
|
"acc_stderr": 0.022658174179374095,
|
|
"em_relaxed": 0.2925,
|
|
"em_relaxed_stderr": 0.022774020616430368
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"acc": 0.23166023166023167,
|
|
"acc_stderr": 0.02626591571601146,
|
|
"em_relaxed": 0.23166023166023167,
|
|
"em_relaxed_stderr": 0.02626591571601146
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": {
|
|
"acc": 0.034482758620689655,
|
|
"acc_stderr": 0.034482758620689676
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
|
|
"acc": 0.2413793103448276,
|
|
"acc_stderr": 0.02399654973117778,
|
|
"em_relaxed": 0.2413793103448276,
|
|
"em_relaxed_stderr": 0.02399654973117778
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
|
|
"acc": 0.18666666666666668,
|
|
"acc_stderr": 0.0319208566416966,
|
|
"em_relaxed": 0.18666666666666668,
|
|
"em_relaxed_stderr": 0.0319208566416966
|
|
},
|
|
"sklep.prompt3_nli_3_classes_sk:nli|0": {
|
|
"acc": 0.002398081534772182,
|
|
"acc_stderr": 0.0006915050545011868,
|
|
"em_norm_relaxed": 0.002398081534772182,
|
|
"em_norm_relaxed_stderr": 0.0006915050545011868
|
|
},
|
|
"e_tests.prompt3_qa_sk:fyzika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.014285714285714287,
|
|
"rouge1_stderr": 0.014285714285714287,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.014285714285714287,
|
|
"rougeL_stderr": 0.014285714285714287
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
|
|
"acc": 0.25892857142857145,
|
|
"acc_stderr": 0.041577515398656284,
|
|
"em_relaxed": 0.25892857142857145,
|
|
"em_relaxed_stderr": 0.041577515398656284
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
|
|
"acc": 0.3048780487804878,
|
|
"acc_stderr": 0.03605784583600454,
|
|
"em_relaxed": 0.3048780487804878,
|
|
"em_relaxed_stderr": 0.03605784583600454
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:informatika|0": {
|
|
"acc": 0.224,
|
|
"acc_stderr": 0.03744070581082714,
|
|
"em_relaxed": 0.224,
|
|
"em_relaxed_stderr": 0.03744070581082714
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
|
|
"acc": 0.28498074454428757,
|
|
"acc_stderr": 0.016183663085909714,
|
|
"em_relaxed": 0.28498074454428757,
|
|
"em_relaxed_stderr": 0.016183663085909714
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
|
|
"acc": 0.25838926174496646,
|
|
"acc_stderr": 0.01794596235855172,
|
|
"em_relaxed": 0.26174496644295303,
|
|
"em_relaxed_stderr": 0.01802120781376133
|
|
},
|
|
"arc.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.24550898203592814,
|
|
"acc_stderr": 0.012593293013246194,
|
|
"em_relaxed": 0.24550898203592814,
|
|
"em_relaxed_stderr": 0.012593293013246194
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"acc": 0.3191489361702128,
|
|
"acc_stderr": 0.030472973363380035,
|
|
"em_relaxed": 0.3191489361702128,
|
|
"em_relaxed_stderr": 0.030472973363380035
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:vsp|0": {
|
|
"acc": 0.24514563106796117,
|
|
"acc_stderr": 0.014994906005686719,
|
|
"em_relaxed": 0.24514563106796117,
|
|
"em_relaxed_stderr": 0.014994906005686719
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": {
|
|
"acc": 0.03571428571428571,
|
|
"acc_stderr": 0.025023180348048964
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.27074235807860264,
|
|
"acc_stderr": 0.02078550104939707,
|
|
"em_relaxed": 0.27074235807860264,
|
|
"em_relaxed_stderr": 0.02078550104939707
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
|
|
"acc": 0.2631578947368421,
|
|
"acc_stderr": 0.03377310252209194,
|
|
"em_relaxed": 0.2631578947368421,
|
|
"em_relaxed_stderr": 0.03377310252209194
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"acc": 0.26262626262626265,
|
|
"acc_stderr": 0.04445287676983945,
|
|
"em_relaxed": 0.26262626262626265,
|
|
"em_relaxed_stderr": 0.04445287676983945
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
|
|
"acc": 0.26785714285714285,
|
|
"acc_stderr": 0.04203277291467762,
|
|
"em_relaxed": 0.2857142857142857,
|
|
"em_relaxed_stderr": 0.04287858751340456
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
|
|
"acc": 0.16216216216216217,
|
|
"acc_stderr": 0.02479468591119148,
|
|
"em_relaxed": 0.16666666666666666,
|
|
"em_relaxed_stderr": 0.02506903489984114
|
|
},
|
|
"belebele.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.2577777777777778,
|
|
"acc_stderr": 0.014588474089651667,
|
|
"em_relaxed": 0.2577777777777778,
|
|
"em_relaxed_stderr": 0.014588474089651667
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
|
|
"acc": 0.26785714285714285,
|
|
"acc_stderr": 0.04203277291467762,
|
|
"em_relaxed": 0.2767857142857143,
|
|
"em_relaxed_stderr": 0.042466243366976256
|
|
},
|
|
"slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.024691358024691357,
|
|
"qf1_stderr": 0.024691358024691353,
|
|
"rouge1": 0.01851851851851852,
|
|
"rouge1_stderr": 0.01851851851851852,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.01851851851851852,
|
|
"rougeL_stderr": 0.01851851851851852
|
|
},
|
|
"movie_review.prompt2_sentiment_binary_sk|0": {
|
|
"acc": 0.13097609561752988,
|
|
"acc_stderr": 0.007530749388005803,
|
|
"em_norm_relaxed": 0.13645418326693226,
|
|
"em_norm_relaxed_stderr": 0.007662357895460915
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.13043478260869565,
|
|
"acc_stderr": 0.04084071463598805,
|
|
"em_relaxed": 0.14492753623188406,
|
|
"em_relaxed_stderr": 0.042689636687312106
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
|
|
"acc": 0.20930232558139536,
|
|
"acc_stderr": 0.03110958390976464,
|
|
"em_relaxed": 0.20930232558139536,
|
|
"em_relaxed_stderr": 0.03110958390976464
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"acc": 0.2277992277992278,
|
|
"acc_stderr": 0.026111473881583542,
|
|
"em_relaxed": 0.2277992277992278,
|
|
"em_relaxed_stderr": 0.026111473881583542
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
|
|
"acc": 0.2734274711168164,
|
|
"acc_stderr": 0.015979778720725094,
|
|
"em_relaxed": 0.2734274711168164,
|
|
"em_relaxed_stderr": 0.015979778720725094
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"acc": 0.22784810126582278,
|
|
"acc_stderr": 0.033475282640609255,
|
|
"em_relaxed": 0.22784810126582278,
|
|
"em_relaxed_stderr": 0.033475282640609255
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
|
|
"acc": 0.29906542056074764,
|
|
"acc_stderr": 0.044470182376718334,
|
|
"em_relaxed": 0.29906542056074764,
|
|
"em_relaxed_stderr": 0.044470182376718334
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
|
|
"acc": 0.21568627450980393,
|
|
"acc_stderr": 0.04092563958237656,
|
|
"em_relaxed": 0.21568627450980393,
|
|
"em_relaxed_stderr": 0.04092563958237656
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"acc": 0.2064343163538874,
|
|
"acc_stderr": 0.020985093876713833,
|
|
"em_relaxed": 0.2064343163538874,
|
|
"em_relaxed_stderr": 0.020985093876713833
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": {
|
|
"acc": 0.25203252032520324,
|
|
"acc_stderr": 0.022633188466788714,
|
|
"em_relaxed": 0.25203252032520324,
|
|
"em_relaxed_stderr": 0.022633188466788714
|
|
},
|
|
"e_tests.prompt3_qa_sk:dejepis|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:mat|0": {
|
|
"acc": 0.1885245901639344,
|
|
"acc_stderr": 0.020472706124868213,
|
|
"em_relaxed": 0.1912568306010929,
|
|
"em_relaxed_stderr": 0.020585781457546286
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": {
|
|
"acc": 0.02857142857142857,
|
|
"acc_stderr": 0.028571428571428574
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:mat|0": {
|
|
"acc": 0.31420765027322406,
|
|
"acc_stderr": 0.024297313836707593,
|
|
"em_relaxed": 0.31693989071038253,
|
|
"em_relaxed_stderr": 0.024354066150157113
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
|
|
"acc": 0.22340425531914893,
|
|
"acc_stderr": 0.024847921358063962,
|
|
"em_relaxed": 0.23404255319148937,
|
|
"em_relaxed_stderr": 0.025257861359432414
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": {
|
|
"acc": 0.14285714285714285,
|
|
"acc_stderr": 0.03431317581537582,
|
|
"em_relaxed": 0.14285714285714285,
|
|
"em_relaxed_stderr": 0.03431317581537582
|
|
},
|
|
"slovakcopa.prompt2_copa_sk|0": {
|
|
"acc": 0.392,
|
|
"acc_stderr": 0.021854684955611256,
|
|
"em_relaxed": 0.394,
|
|
"em_relaxed_stderr": 0.021874299301689253
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"acc": 0.22962962962962963,
|
|
"acc_stderr": 0.036333844140734636,
|
|
"em_relaxed": 0.22962962962962963,
|
|
"em_relaxed_stderr": 0.036333844140734636
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.02857142857142857,
|
|
"acc_stderr": 0.014130710990583526
|
|
},
|
|
"belebele.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.27111111111111114,
|
|
"acc_stderr": 0.014826016446581962,
|
|
"em_relaxed": 0.27111111111111114,
|
|
"em_relaxed_stderr": 0.014826016446581962
|
|
},
|
|
"skj1.prompt1_skj1_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_relaxed": 0.01,
|
|
"em_relaxed_stderr": 0.009999999999999997
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
|
|
"acc": 0.26,
|
|
"acc_stderr": 0.0440844002276808,
|
|
"em_relaxed": 0.26,
|
|
"em_relaxed_stderr": 0.0440844002276808
|
|
},
|
|
"driving_test.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.2738944365192582,
|
|
"acc_stderr": 0.01685553212123784,
|
|
"em_relaxed": 0.2738944365192582,
|
|
"em_relaxed_stderr": 0.01685553212123784
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:aj|0": {
|
|
"acc": 0.3,
|
|
"acc_stderr": 0.15275252316519464,
|
|
"em_relaxed": 0.3,
|
|
"em_relaxed_stderr": 0.15275252316519464
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
|
|
"acc": 0.26,
|
|
"acc_stderr": 0.04408440022768078,
|
|
"em_relaxed": 0.26,
|
|
"em_relaxed_stderr": 0.04408440022768078
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:bio|0": {
|
|
"acc": 0.21238938053097345,
|
|
"acc_stderr": 0.022246609446382862,
|
|
"em_relaxed": 0.21238938053097345,
|
|
"em_relaxed_stderr": 0.022246609446382862
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
|
|
"acc": 0.24,
|
|
"acc_stderr": 0.04292346959909283,
|
|
"em_relaxed": 0.24,
|
|
"em_relaxed_stderr": 0.04292346959909283
|
|
},
|
|
"skj1.prompt3_skj1_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_relaxed": 0.0,
|
|
"em_relaxed_stderr": 0.0
|
|
},
|
|
"skj1.prompt2_skj1_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_relaxed": 0.0,
|
|
"em_relaxed_stderr": 0.0
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": {
|
|
"acc": 0.02857142857142857,
|
|
"acc_stderr": 0.028571428571428574
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
|
|
"acc": 0.22033898305084745,
|
|
"acc_stderr": 0.03831824849223319,
|
|
"em_relaxed": 0.22033898305084745,
|
|
"em_relaxed_stderr": 0.03831824849223319
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:dejepis|0": {
|
|
"acc": 0.1875,
|
|
"acc_stderr": 0.05693291124629694,
|
|
"em_relaxed": 0.1875,
|
|
"em_relaxed_stderr": 0.05693291124629694
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
|
|
"acc": 0.23391812865497075,
|
|
"acc_stderr": 0.03246721765117826,
|
|
"em_relaxed": 0.23391812865497075,
|
|
"em_relaxed_stderr": 0.03246721765117826
|
|
},
|
|
"slovak_tests.prompt1_qa_sk:maturita|0": {
|
|
"qem": 0.008849557522123894,
|
|
"qem_stderr": 0.0062436609110668876,
|
|
"qf1": 0.007079646017699116,
|
|
"qf1_stderr": 0.0038256704785137783,
|
|
"rouge1": 0.014790713408591503,
|
|
"rouge1_stderr": 0.004864012429457399,
|
|
"rouge2": 0.003448518050287962,
|
|
"rouge2_stderr": 0.002428770392002376,
|
|
"rougeL": 0.014790713408591503,
|
|
"rougeL_stderr": 0.004864012429457399
|
|
},
|
|
"sklep.prompt1_qa_sk:question-answering|0": {
|
|
"qem": 0.05248878221851195,
|
|
"qem_stderr": 0.002278228840555749,
|
|
"qf1": 0.1424834912487429,
|
|
"qf1_stderr": 0.0027244148341408567
|
|
},
|
|
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.2725,
|
|
"acc_stderr": 0.022290166077429768,
|
|
"em_relaxed": 0.2725,
|
|
"em_relaxed_stderr": 0.022290166077429768
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
|
|
"acc": 0.24846625766871167,
|
|
"acc_stderr": 0.02396990269668754,
|
|
"em_relaxed": 0.24846625766871167,
|
|
"em_relaxed_stderr": 0.02396990269668754
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
|
|
"acc": 0.18468468468468469,
|
|
"acc_stderr": 0.026102500004017648,
|
|
"em_relaxed": 0.18468468468468469,
|
|
"em_relaxed_stderr": 0.026102500004017648
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.03571428571428571,
|
|
"acc_stderr": 0.03571428571428572
|
|
},
|
|
"sklep.prompt1_nli_3_classes_sk:nli|0": {
|
|
"acc": 0.004396482813749001,
|
|
"acc_stderr": 0.0009353639970831211,
|
|
"em_norm_relaxed": 0.004996003197442046,
|
|
"em_norm_relaxed_stderr": 0.0009968011147772275
|
|
},
|
|
"e_tests.prompt2_qa_sk:obcianska-nauka|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.020408163265306124,
|
|
"rouge1_stderr": 0.020408163265306124,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.020408163265306124,
|
|
"rougeL_stderr": 0.020408163265306124
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
|
|
"acc": 0.24358974358974358,
|
|
"acc_stderr": 0.028120966503914394,
|
|
"em_relaxed": 0.24358974358974358,
|
|
"em_relaxed_stderr": 0.028120966503914394
|
|
},
|
|
"financial_exams.prompt3_completion_sk|0": {
|
|
"acc": 0.18590704647676162,
|
|
"acc_stderr": 0.010655402235931219,
|
|
"em_relaxed": 0.18590704647676162,
|
|
"em_relaxed_stderr": 0.010655402235931219
|
|
},
|
|
"arc.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.24721984602224123,
|
|
"acc_stderr": 0.012622759999132588,
|
|
"em_relaxed": 0.24721984602224123,
|
|
"em_relaxed_stderr": 0.012622759999132588
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": {
|
|
"acc": 0.30414746543778803,
|
|
"acc_stderr": 0.0313021257639245,
|
|
"em_relaxed": 0.30414746543778803,
|
|
"em_relaxed_stderr": 0.0313021257639245
|
|
},
|
|
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"acc": 0.09213051823416507,
|
|
"acc_stderr": 0.00896371626415267,
|
|
"em_norm_relaxed": 0.09404990403071017,
|
|
"em_norm_relaxed_stderr": 0.009047028350861394
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:matematika|0": {
|
|
"acc": 0.15384615384615385,
|
|
"acc_stderr": 0.05852968540405248,
|
|
"em_relaxed": 0.15384615384615385,
|
|
"em_relaxed_stderr": 0.05852968540405248
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
|
|
"acc": 0.20567375886524822,
|
|
"acc_stderr": 0.024112138950471887,
|
|
"em_relaxed": 0.20567375886524822,
|
|
"em_relaxed_stderr": 0.024112138950471887
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
|
|
"acc": 0.2601880877742947,
|
|
"acc_stderr": 0.024603155110002158,
|
|
"em_relaxed": 0.26332288401253917,
|
|
"em_relaxed_stderr": 0.02469842907233297
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"acc": 0.1822429906542056,
|
|
"acc_stderr": 0.026451344562470145,
|
|
"em_relaxed": 0.1822429906542056,
|
|
"em_relaxed_stderr": 0.026451344562470145
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"acc": 0.22807017543859648,
|
|
"acc_stderr": 0.056069817847611766,
|
|
"em_relaxed": 0.22807017543859648,
|
|
"em_relaxed_stderr": 0.056069817847611766
|
|
},
|
|
"e_tests.prompt1_qa_sk:fyzika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.014285714285714287,
|
|
"rouge1_stderr": 0.014285714285714287,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.014285714285714287,
|
|
"rougeL_stderr": 0.014285714285714287
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"acc": 0.24242424242424243,
|
|
"acc_stderr": 0.04329004329004326,
|
|
"em_relaxed": 0.24242424242424243,
|
|
"em_relaxed_stderr": 0.04329004329004326
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
|
|
"acc": 0.19,
|
|
"acc_stderr": 0.03942772444036623,
|
|
"em_relaxed": 0.19,
|
|
"em_relaxed_stderr": 0.03942772444036623
|
|
},
|
|
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.2675,
|
|
"acc_stderr": 0.022160484939004153,
|
|
"em_relaxed": 0.27,
|
|
"em_relaxed_stderr": 0.022225772481648733
|
|
},
|
|
"e_tests.prompt2_qa_sk:slovensky-jazyk|0": {
|
|
"qem": 0.0051813471502590676,
|
|
"qem_stderr": 0.00518134715025908,
|
|
"qf1": 0.010905502097211939,
|
|
"qf1_stderr": 0.0062654994299230354,
|
|
"rouge1": 0.015304902351534477,
|
|
"rouge1_stderr": 0.006747971135704163,
|
|
"rouge2": 0.0072538860103626935,
|
|
"rouge2_stderr": 0.005570450232424934,
|
|
"rougeL": 0.015304902351534477,
|
|
"rougeL_stderr": 0.006747971135704163
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": {
|
|
"acc": 0.30414746543778803,
|
|
"acc_stderr": 0.0313021257639245,
|
|
"em_relaxed": 0.30414746543778803,
|
|
"em_relaxed_stderr": 0.0313021257639245
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
|
|
"acc": 0.3515981735159817,
|
|
"acc_stderr": 0.03233829906905826,
|
|
"em_relaxed": 0.3515981735159817,
|
|
"em_relaxed_stderr": 0.03233829906905826
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"acc": 0.2706766917293233,
|
|
"acc_stderr": 0.027293707912941223,
|
|
"em_relaxed": 0.2706766917293233,
|
|
"em_relaxed_stderr": 0.027293707912941223
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
|
|
"acc": 0.27522935779816515,
|
|
"acc_stderr": 0.042976970405367555,
|
|
"em_relaxed": 0.28440366972477066,
|
|
"em_relaxed_stderr": 0.043409998674551635
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": {
|
|
"acc": 0.2791327913279133,
|
|
"acc_stderr": 0.02338348056654487,
|
|
"em_relaxed": 0.2791327913279133,
|
|
"em_relaxed_stderr": 0.02338348056654487
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": {
|
|
"acc": 0.26666666666666666,
|
|
"acc_stderr": 0.04336290903919941,
|
|
"em_relaxed": 0.26666666666666666,
|
|
"em_relaxed_stderr": 0.04336290903919941
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
|
|
"acc": 0.1513157894736842,
|
|
"acc_stderr": 0.029162631596843982,
|
|
"em_relaxed": 0.1513157894736842,
|
|
"em_relaxed_stderr": 0.029162631596843982
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"acc": 0.2828282828282828,
|
|
"acc_stderr": 0.04549461117008498,
|
|
"em_relaxed": 0.2828282828282828,
|
|
"em_relaxed_stderr": 0.04549461117008498
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:biologia|0": {
|
|
"acc": 0.22988505747126436,
|
|
"acc_stderr": 0.045371581852507746,
|
|
"em_relaxed": 0.22988505747126436,
|
|
"em_relaxed_stderr": 0.045371581852507746
|
|
},
|
|
"e_tests.prompt1_qa_sk:chemia|0": {
|
|
"qem": 0.018518518518518517,
|
|
"qem_stderr": 0.018518518518518514,
|
|
"qf1": 0.012345679012345678,
|
|
"qf1_stderr": 0.012345679012345675,
|
|
"rouge1": 0.02580246913580247,
|
|
"rouge1_stderr": 0.011283149325161083,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.02580246913580247,
|
|
"rougeL_stderr": 0.011283149325161083
|
|
},
|
|
"e_tests.prompt2_qa_sk:biologia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.019047619047619046,
|
|
"qf1_stderr": 0.019047619047619035,
|
|
"rouge1": 0.02857142857142857,
|
|
"rouge1_stderr": 0.01990372988626389,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.02857142857142857,
|
|
"rougeL_stderr": 0.01990372988626389
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
|
|
"acc": 0.3,
|
|
"acc_stderr": 0.046056618647183814,
|
|
"em_relaxed": 0.3,
|
|
"em_relaxed_stderr": 0.046056618647183814
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
|
|
"acc": 0.27,
|
|
"acc_stderr": 0.044619604333847394,
|
|
"em_relaxed": 0.27,
|
|
"em_relaxed_stderr": 0.044619604333847394
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.032769276820761584,
|
|
"em_relaxed": 0.20666666666666667,
|
|
"em_relaxed_stderr": 0.033171868444173455
|
|
},
|
|
"slovak_tests.prompt2_qa_sk:monitor9|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:mat|0": {
|
|
"acc": 0.23770491803278687,
|
|
"acc_stderr": 0.022280975973791882,
|
|
"em_relaxed": 0.24316939890710382,
|
|
"em_relaxed_stderr": 0.022454705846132374
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
|
|
"acc": 0.18686868686868688,
|
|
"acc_stderr": 0.027772533334218974,
|
|
"em_relaxed": 0.1919191919191919,
|
|
"em_relaxed_stderr": 0.028057791672989024
|
|
},
|
|
"multi_wiki_qa.prompt1_qa_sk|0": {
|
|
"qem": 0.0756453872323394,
|
|
"qem_stderr": 0.003741099997048089,
|
|
"qf1": 0.12194744826062183,
|
|
"qf1_stderr": 0.0038908205119327767
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"acc": 0.29,
|
|
"acc_stderr": 0.045604802157206845,
|
|
"em_relaxed": 0.29,
|
|
"em_relaxed_stderr": 0.045604802157206845
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.03571428571428571,
|
|
"acc_stderr": 0.03571428571428572
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
|
|
"acc": 0.18018018018018017,
|
|
"acc_stderr": 0.02585333527853839,
|
|
"em_relaxed": 0.18018018018018017,
|
|
"em_relaxed_stderr": 0.02585333527853839
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:vsp|0": {
|
|
"acc": 0.25849514563106796,
|
|
"acc_stderr": 0.015261010021381638,
|
|
"em_relaxed": 0.2621359223300971,
|
|
"em_relaxed_stderr": 0.01533033114193304
|
|
},
|
|
"e_tests.prompt2_qa_sk:dejepis|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"acc": 0.16580310880829016,
|
|
"acc_stderr": 0.026839845022314415,
|
|
"em_relaxed": 0.16580310880829016,
|
|
"em_relaxed_stderr": 0.026839845022314415
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
|
|
"acc": 0.21,
|
|
"acc_stderr": 0.040936018074033256,
|
|
"em_relaxed": 0.21,
|
|
"em_relaxed_stderr": 0.040936018074033256
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"acc": 0.23232323232323232,
|
|
"acc_stderr": 0.04266016017054689,
|
|
"em_relaxed": 0.24242424242424243,
|
|
"em_relaxed_stderr": 0.04329004329004326
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
|
|
"acc": 0.2463768115942029,
|
|
"acc_stderr": 0.025984280443996907,
|
|
"em_relaxed": 0.2463768115942029,
|
|
"em_relaxed_stderr": 0.025984280443996907
|
|
},
|
|
"e_tests.prompt1_qa_sk:informatika|0": {
|
|
"qem": 0.038461538461538464,
|
|
"qem_stderr": 0.021915495473007307,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0032865282865282863,
|
|
"rouge1_stderr": 0.0022973226212271926,
|
|
"rouge2": 0.0008547008547008547,
|
|
"rouge2_stderr": 0.0008547008547008546,
|
|
"rougeL": 0.0032865282865282863,
|
|
"rougeL_stderr": 0.0022973226212271926
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
|
|
"acc": 0.18,
|
|
"acc_stderr": 0.038612291966536955,
|
|
"em_relaxed": 0.18,
|
|
"em_relaxed_stderr": 0.038612291966536955
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"acc": 0.22119815668202766,
|
|
"acc_stderr": 0.02824082382619082,
|
|
"em_relaxed": 0.2304147465437788,
|
|
"em_relaxed_stderr": 0.0286521112206854
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"acc": 0.22162883845126835,
|
|
"acc_stderr": 0.015186430423507605,
|
|
"em_relaxed": 0.22162883845126835,
|
|
"em_relaxed_stderr": 0.015186430423507605
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": {
|
|
"acc": 0.24744897959183673,
|
|
"acc_stderr": 0.021823400829422215,
|
|
"em_relaxed": 0.24744897959183673,
|
|
"em_relaxed_stderr": 0.021823400829422215
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
|
|
"acc": 0.3,
|
|
"acc_stderr": 0.046056618647183814,
|
|
"em_relaxed": 0.3,
|
|
"em_relaxed_stderr": 0.046056618647183814
|
|
},
|
|
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
|
|
"acc": 0.397172236503856,
|
|
"acc_stderr": 0.01755397820008727,
|
|
"em_relaxed": 0.40359897172236503,
|
|
"em_relaxed_stderr": 0.017600852681572653
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"acc": 0.24561403508771928,
|
|
"acc_stderr": 0.057521390564052664,
|
|
"em_relaxed": 0.24561403508771928,
|
|
"em_relaxed_stderr": 0.057521390564052664
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"acc": 0.21963824289405684,
|
|
"acc_stderr": 0.021072124180856616,
|
|
"em_relaxed": 0.21963824289405684,
|
|
"em_relaxed_stderr": 0.021072124180856616
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
|
|
"acc": 0.2803738317757009,
|
|
"acc_stderr": 0.04362839933570099,
|
|
"em_relaxed": 0.2803738317757009,
|
|
"em_relaxed_stderr": 0.04362839933570099
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:geografia|0": {
|
|
"acc": 0.24489795918367346,
|
|
"acc_stderr": 0.062069005411206336,
|
|
"em_relaxed": 0.24489795918367346,
|
|
"em_relaxed_stderr": 0.062069005411206336
|
|
},
|
|
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
|
|
"acc": 0.13624678663239073,
|
|
"acc_stderr": 0.012306865740559735,
|
|
"em_relaxed": 0.14524421593830333,
|
|
"em_relaxed_stderr": 0.012640374409743442
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
|
|
"acc": 0.21739130434782608,
|
|
"acc_stderr": 0.024872937110754584,
|
|
"em_relaxed": 0.21739130434782608,
|
|
"em_relaxed_stderr": 0.024872937110754584
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"acc": 0.24444444444444444,
|
|
"acc_stderr": 0.03212157057535212,
|
|
"em_relaxed": 0.24444444444444444,
|
|
"em_relaxed_stderr": 0.03212157057535212
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"acc": 0.22830440587449932,
|
|
"acc_stderr": 0.015347207245326627,
|
|
"em_relaxed": 0.22963951935914553,
|
|
"em_relaxed_stderr": 0.015378696047536682
|
|
},
|
|
"e_tests.prompt3_qa_sk:informatika|0": {
|
|
"qem": 0.02564102564102564,
|
|
"qem_stderr": 0.01801282495677459,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0036324786324786326,
|
|
"rouge1_stderr": 0.0025563439430967027,
|
|
"rouge2": 0.0008547008547008547,
|
|
"rouge2_stderr": 0.0008547008547008546,
|
|
"rougeL": 0.0036324786324786326,
|
|
"rougeL_stderr": 0.0025563439430967027
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
|
|
"acc": 0.21753246753246752,
|
|
"acc_stderr": 0.023546481379339165,
|
|
"em_relaxed": 0.21753246753246752,
|
|
"em_relaxed_stderr": 0.023546481379339165
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
|
|
"acc": 0.25,
|
|
"acc_stderr": 0.04351941398892446,
|
|
"em_relaxed": 0.25,
|
|
"em_relaxed_stderr": 0.04351941398892446
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:bio|0": {
|
|
"acc": 0.24778761061946902,
|
|
"acc_stderr": 0.023482897631940607,
|
|
"em_relaxed": 0.26548672566371684,
|
|
"em_relaxed_stderr": 0.024019440114877274
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
|
|
"acc": 0.30275229357798167,
|
|
"acc_stderr": 0.04421049631437605,
|
|
"em_relaxed": 0.30275229357798167,
|
|
"em_relaxed_stderr": 0.04421049631437605
|
|
},
|
|
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
|
|
"acc": 0.42287917737789205,
|
|
"acc_stderr": 0.017722745992252577,
|
|
"em_relaxed": 0.42287917737789205,
|
|
"em_relaxed_stderr": 0.017722745992252577
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
|
|
"acc": 0.17647058823529413,
|
|
"acc_stderr": 0.0379328118530781,
|
|
"em_relaxed": 0.18627450980392157,
|
|
"em_relaxed_stderr": 0.03873958714149352
|
|
},
|
|
"financial_exams.prompt1_completion_sk|0": {
|
|
"acc": 0.18290854572713644,
|
|
"acc_stderr": 0.010588568678863373,
|
|
"em_relaxed": 0.18290854572713644,
|
|
"em_relaxed_stderr": 0.010588568678863373
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
|
|
"acc": 0.3211009174311927,
|
|
"acc_stderr": 0.044927422774799206,
|
|
"em_relaxed": 0.3211009174311927,
|
|
"em_relaxed_stderr": 0.044927422774799206
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
|
|
"acc": 0.20666666666666667,
|
|
"acc_stderr": 0.033171868444173455,
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.033936379482039564
|
|
},
|
|
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
|
|
"acc": 0.0932524639878696,
|
|
"acc_stderr": 0.00800968883832857,
|
|
"em_norm_relaxed": 0.09401061410159212,
|
|
"em_norm_relaxed_stderr": 0.008038819818872478
|
|
},
|
|
"e_tests.prompt3_qa_sk:geografia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
|
|
"acc": 0.233502538071066,
|
|
"acc_stderr": 0.030218515643241007,
|
|
"em_relaxed": 0.233502538071066,
|
|
"em_relaxed_stderr": 0.030218515643241007
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
|
|
"acc": 0.21212121212121213,
|
|
"acc_stderr": 0.041296069325408906,
|
|
"em_relaxed": 0.21212121212121213,
|
|
"em_relaxed_stderr": 0.041296069325408906
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
|
|
"acc": 0.23232323232323232,
|
|
"acc_stderr": 0.042660160170546885,
|
|
"em_relaxed": 0.23232323232323232,
|
|
"em_relaxed_stderr": 0.042660160170546885
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
|
|
"acc": 0.26785714285714285,
|
|
"acc_stderr": 0.04203277291467762,
|
|
"em_relaxed": 0.2767857142857143,
|
|
"em_relaxed_stderr": 0.04246624336697624
|
|
},
|
|
"slovakcopa.prompt3_copa_sk|0": {
|
|
"acc": 0.016,
|
|
"acc_stderr": 0.0056170367814719815,
|
|
"em_relaxed": 0.016,
|
|
"em_relaxed_stderr": 0.0056170367814719815
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.02142857142857143,
|
|
"acc_stderr": 0.012282463299882659
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
|
|
"acc": 0.27102803738317754,
|
|
"acc_stderr": 0.043172737765666686,
|
|
"em_relaxed": 0.27102803738317754,
|
|
"em_relaxed_stderr": 0.043172737765666686
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": {
|
|
"acc": 0.2903225806451613,
|
|
"acc_stderr": 0.030884745411494743,
|
|
"em_relaxed": 0.2903225806451613,
|
|
"em_relaxed_stderr": 0.030884745411494743
|
|
},
|
|
"financial_exams.prompt2_completion_sk|0": {
|
|
"acc": 0.1896551724137931,
|
|
"acc_stderr": 0.010737475929667924,
|
|
"em_relaxed": 0.1904047976011994,
|
|
"em_relaxed_stderr": 0.010753697898553788
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"acc": 0.19170984455958548,
|
|
"acc_stderr": 0.02840895362624528,
|
|
"em_relaxed": 0.19170984455958548,
|
|
"em_relaxed_stderr": 0.02840895362624528
|
|
},
|
|
"e_tests.prompt1_qa_sk:slovensky-jazyk|0": {
|
|
"qem": 0.0051813471502590676,
|
|
"qem_stderr": 0.00518134715025908,
|
|
"qf1": 0.01526314790563495,
|
|
"qf1_stderr": 0.006730183842906608,
|
|
"rouge1": 0.01724455022545386,
|
|
"rouge1_stderr": 0.006851608530741241,
|
|
"rouge2": 0.0073447868375602215,
|
|
"rouge2_stderr": 0.005570575389175516,
|
|
"rougeL": 0.01724455022545386,
|
|
"rougeL_stderr": 0.006851608530741241
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"acc": 0.22962962962962963,
|
|
"acc_stderr": 0.03633384414073462,
|
|
"em_relaxed": 0.22962962962962963,
|
|
"em_relaxed_stderr": 0.03633384414073462
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
|
|
"acc": 0.2862644415917843,
|
|
"acc_stderr": 0.01620550507370852,
|
|
"em_relaxed": 0.28754813863928114,
|
|
"em_relaxed_stderr": 0.01622718713515171
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
|
|
"acc": 0.2537313432835821,
|
|
"acc_stderr": 0.037731914327133756,
|
|
"em_relaxed": 0.2537313432835821,
|
|
"em_relaxed_stderr": 0.037731914327133756
|
|
},
|
|
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"acc": 0.04702495201535509,
|
|
"acc_stderr": 0.006561144166938836,
|
|
"em_norm_relaxed": 0.05566218809980806,
|
|
"em_norm_relaxed_stderr": 0.007105889705516419
|
|
},
|
|
"toxic_sk.prompt3_toxicity_sk|0": {
|
|
"acc": 0.007945516458569807,
|
|
"acc_stderr": 0.0029928675111869866,
|
|
"em_norm_relaxed": 0.007945516458569807,
|
|
"em_norm_relaxed_stderr": 0.0029928675111869866
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": {
|
|
"acc": 0.017857142857142856,
|
|
"acc_stderr": 0.017857142857142863
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"acc": 0.24561403508771928,
|
|
"acc_stderr": 0.05752139056405266,
|
|
"em_relaxed": 0.24561403508771928,
|
|
"em_relaxed_stderr": 0.05752139056405266
|
|
},
|
|
"slovak_tests.prompt3_qa_sk:maturita|0": {
|
|
"qem": 0.01327433628318584,
|
|
"qem_stderr": 0.007629803632081178,
|
|
"qf1": 0.011971484759095378,
|
|
"qf1_stderr": 0.0058019008534905304,
|
|
"rouge1": 0.014379696684114138,
|
|
"rouge1_stderr": 0.0046830153944730545,
|
|
"rouge2": 0.004080629301868239,
|
|
"rouge2_stderr": 0.002569082757498883,
|
|
"rougeL": 0.014379696684114138,
|
|
"rougeL_stderr": 0.0046830153944730545
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.04020151261036845,
|
|
"em_relaxed": 0.2,
|
|
"em_relaxed_stderr": 0.04020151261036845
|
|
},
|
|
"slovak_tests.prompt2_qa_sk:maturita|0": {
|
|
"qem": 0.008849557522123894,
|
|
"qem_stderr": 0.0062436609110668876,
|
|
"qf1": 0.0057030481809242865,
|
|
"qf1_stderr": 0.0035655859886822355,
|
|
"rouge1": 0.015324943300924803,
|
|
"rouge1_stderr": 0.0047498192758248054,
|
|
"rouge2": 0.0022123893805309734,
|
|
"rouge2_stderr": 0.002212389380530974,
|
|
"rougeL": 0.015324943300924803,
|
|
"rougeL_stderr": 0.0047498192758248054
|
|
},
|
|
"slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.036387264457439894,
|
|
"qf1_stderr": 0.025966525279646633,
|
|
"rouge1": 0.027777777777777776,
|
|
"rouge1_stderr": 0.019641855032959652,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.027777777777777776,
|
|
"rougeL_stderr": 0.019641855032959652
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:dejepis|0": {
|
|
"acc": 0.1875,
|
|
"acc_stderr": 0.05693291124629694,
|
|
"em_relaxed": 0.1875,
|
|
"em_relaxed_stderr": 0.05693291124629694
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": {
|
|
"acc": 0.2653061224489796,
|
|
"acc_stderr": 0.022327416188566384,
|
|
"em_relaxed": 0.2653061224489796,
|
|
"em_relaxed_stderr": 0.022327416188566384
|
|
},
|
|
"e_tests.prompt2_qa_sk:chemia|0": {
|
|
"qem": 0.037037037037037035,
|
|
"qem_stderr": 0.025940896147128693,
|
|
"qf1": 0.012345679012345678,
|
|
"qf1_stderr": 0.012345679012345675,
|
|
"rouge1": 0.02580246913580247,
|
|
"rouge1_stderr": 0.011283149325161083,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.02580246913580247,
|
|
"rougeL_stderr": 0.011283149325161083
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.26419213973799127,
|
|
"acc_stderr": 0.020624529862085718,
|
|
"em_relaxed": 0.26419213973799127,
|
|
"em_relaxed_stderr": 0.020624529862085718
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
|
|
"acc": 0.22402597402597402,
|
|
"acc_stderr": 0.02379597980086175,
|
|
"em_relaxed": 0.22402597402597402,
|
|
"em_relaxed_stderr": 0.02379597980086175
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:fyzika|0": {
|
|
"acc": 0.22727272727272727,
|
|
"acc_stderr": 0.0914486154730632,
|
|
"em_relaxed": 0.22727272727272727,
|
|
"em_relaxed_stderr": 0.0914486154730632
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
|
|
"acc": 0.19318181818181818,
|
|
"acc_stderr": 0.024344064423295675,
|
|
"em_relaxed": 0.19318181818181818,
|
|
"em_relaxed_stderr": 0.024344064423295675
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": {
|
|
"acc": 0.1,
|
|
"acc_stderr": 0.06882472016116853
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
|
|
"acc": 0.29,
|
|
"acc_stderr": 0.045604802157206845,
|
|
"em_relaxed": 0.3,
|
|
"em_relaxed_stderr": 0.046056618647183814
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"acc": 0.24812030075187969,
|
|
"acc_stderr": 0.02653275776026344,
|
|
"em_relaxed": 0.2518796992481203,
|
|
"em_relaxed_stderr": 0.026666091559782525
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
|
|
"acc": 0.2605633802816901,
|
|
"acc_stderr": 0.03696558732615874,
|
|
"em_relaxed": 0.2605633802816901,
|
|
"em_relaxed_stderr": 0.03696558732615874
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": {
|
|
"acc": 0.02857142857142857,
|
|
"acc_stderr": 0.028571428571428574
|
|
},
|
|
"slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.024691358024691357,
|
|
"qf1_stderr": 0.024691358024691353,
|
|
"rouge1": 0.01851851851851852,
|
|
"rouge1_stderr": 0.01851851851851852,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.01851851851851852,
|
|
"rougeL_stderr": 0.01851851851851852
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
|
|
"acc": 0.23134328358208955,
|
|
"acc_stderr": 0.036565274697599,
|
|
"em_relaxed": 0.23134328358208955,
|
|
"em_relaxed_stderr": 0.036565274697599
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
|
|
"acc": 0.26,
|
|
"acc_stderr": 0.04408440022768078,
|
|
"em_relaxed": 0.26,
|
|
"em_relaxed_stderr": 0.04408440022768078
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"acc": 0.2088607594936709,
|
|
"acc_stderr": 0.03244180901572427,
|
|
"em_relaxed": 0.2088607594936709,
|
|
"em_relaxed_stderr": 0.03244180901572427
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
|
|
"acc": 0.31,
|
|
"acc_stderr": 0.04648231987117316,
|
|
"em_relaxed": 0.31,
|
|
"em_relaxed_stderr": 0.04648231987117316
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:aj|0": {
|
|
"acc": 0.3,
|
|
"acc_stderr": 0.15275252316519464,
|
|
"em_relaxed": 0.3,
|
|
"em_relaxed_stderr": 0.15275252316519464
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
|
|
"acc": 0.21739130434782608,
|
|
"acc_stderr": 0.038631448549506,
|
|
"em_relaxed": 0.25217391304347825,
|
|
"em_relaxed_stderr": 0.04067222754154718
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
|
|
"acc": 0.2457627118644068,
|
|
"acc_stderr": 0.03980329854920433,
|
|
"em_relaxed": 0.2457627118644068,
|
|
"em_relaxed_stderr": 0.03980329854920433
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:chemia|0": {
|
|
"acc": 0.17857142857142858,
|
|
"acc_stderr": 0.05164277182008721,
|
|
"em_relaxed": 0.17857142857142858,
|
|
"em_relaxed_stderr": 0.05164277182008721
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"acc": 0.2386058981233244,
|
|
"acc_stderr": 0.022099063590345355,
|
|
"em_relaxed": 0.2386058981233244,
|
|
"em_relaxed_stderr": 0.022099063590345355
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.27510917030567683,
|
|
"acc_stderr": 0.020889629452003455,
|
|
"em_relaxed": 0.27510917030567683,
|
|
"em_relaxed_stderr": 0.020889629452003455
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
|
|
"acc": 0.17763157894736842,
|
|
"acc_stderr": 0.03110318238312337,
|
|
"em_relaxed": 0.18421052631578946,
|
|
"em_relaxed_stderr": 0.031546980450822305
|
|
},
|
|
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
|
|
"acc": 0.3525398028809704,
|
|
"acc_stderr": 0.01315990975593033,
|
|
"em_norm_relaxed": 0.35329795299469297,
|
|
"em_norm_relaxed_stderr": 0.013166337192115686
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
|
|
"acc": 0.21641791044776118,
|
|
"acc_stderr": 0.035707794739409195,
|
|
"em_relaxed": 0.22388059701492538,
|
|
"em_relaxed_stderr": 0.03614487092173135
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:matematika|0": {
|
|
"acc": 0.1282051282051282,
|
|
"acc_stderr": 0.05423355275954149,
|
|
"em_relaxed": 0.1282051282051282,
|
|
"em_relaxed_stderr": 0.05423355275954149
|
|
},
|
|
"sklep.prompt3_qa_sk:question-answering|0": {
|
|
"qem": 0.05123656475007826,
|
|
"qem_stderr": 0.0022523759848186195,
|
|
"qf1": 0.1417299858339243,
|
|
"qf1_stderr": 0.002706574024592746
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": {
|
|
"acc": 0.022727272727272728,
|
|
"acc_stderr": 0.022727272727272728
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
|
|
"acc": 0.2077922077922078,
|
|
"acc_stderr": 0.023156077580247444,
|
|
"em_relaxed": 0.2077922077922078,
|
|
"em_relaxed_stderr": 0.023156077580247444
|
|
},
|
|
"toxic_sk.prompt2_toxicity_sk|0": {
|
|
"acc": 0.23609534619750283,
|
|
"acc_stderr": 0.014316008748210712,
|
|
"em_norm_relaxed": 0.23723041997729852,
|
|
"em_norm_relaxed_stderr": 0.014339715502704882
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
|
|
"acc": 0.24770642201834864,
|
|
"acc_stderr": 0.041538463936410155,
|
|
"em_relaxed": 0.24770642201834864,
|
|
"em_relaxed_stderr": 0.041538463936410155
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.03571428571428571,
|
|
"acc_stderr": 0.03571428571428572
|
|
},
|
|
"senti_sk.prompt3_sentiment_binary_sk|0": {
|
|
"acc": 0.7416845493562232,
|
|
"acc_stderr": 0.007169768138157722,
|
|
"em_norm_relaxed": 0.7475858369098712,
|
|
"em_norm_relaxed_stderr": 0.007115537241408355
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"acc": 0.22758620689655173,
|
|
"acc_stderr": 0.03493950380131184,
|
|
"em_relaxed": 0.22758620689655173,
|
|
"em_relaxed_stderr": 0.03493950380131184
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
|
|
"acc": 0.1856060606060606,
|
|
"acc_stderr": 0.023973722216817445,
|
|
"em_relaxed": 0.1856060606060606,
|
|
"em_relaxed_stderr": 0.023973722216817445
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:informatika|0": {
|
|
"acc": 0.232,
|
|
"acc_stderr": 0.03790650637827825,
|
|
"em_relaxed": 0.232,
|
|
"em_relaxed_stderr": 0.03790650637827825
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
|
|
"acc": 0.27,
|
|
"acc_stderr": 0.044619604333847394,
|
|
"em_relaxed": 0.27,
|
|
"em_relaxed_stderr": 0.044619604333847394
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"acc": 0.27,
|
|
"acc_stderr": 0.044619604333847394,
|
|
"em_relaxed": 0.27,
|
|
"em_relaxed_stderr": 0.044619604333847394
|
|
},
|
|
"toxic_sk.prompt1_toxicity_sk|0": {
|
|
"acc": 0.3530079455164586,
|
|
"acc_stderr": 0.01611017944424598,
|
|
"em_norm_relaxed": 0.35414301929625425,
|
|
"em_norm_relaxed_stderr": 0.016121898615772852
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
|
|
"acc": 0.2651006711409396,
|
|
"acc_stderr": 0.018095094525450207,
|
|
"em_relaxed": 0.26677852348993286,
|
|
"em_relaxed_stderr": 0.018131533555282176
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"acc": 0.23448275862068965,
|
|
"acc_stderr": 0.035306258743465914,
|
|
"em_relaxed": 0.23448275862068965,
|
|
"em_relaxed_stderr": 0.035306258743465914
|
|
},
|
|
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
|
|
"acc": 0.1956027293404094,
|
|
"acc_stderr": 0.010926096810556464,
|
|
"em_norm_relaxed": 0.1956027293404094,
|
|
"em_norm_relaxed_stderr": 0.010926096810556464
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:che|0": {
|
|
"acc": 0.24796747967479674,
|
|
"acc_stderr": 0.027588788664854164,
|
|
"em_relaxed": 0.24796747967479674,
|
|
"em_relaxed_stderr": 0.027588788664854164
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"acc": 0.21717171717171718,
|
|
"acc_stderr": 0.029376616484945637,
|
|
"em_relaxed": 0.21717171717171718,
|
|
"em_relaxed_stderr": 0.029376616484945637
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"acc": 0.2222222222222222,
|
|
"acc_stderr": 0.031073844843659416,
|
|
"em_relaxed": 0.2222222222222222,
|
|
"em_relaxed_stderr": 0.031073844843659416
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:biologia|0": {
|
|
"acc": 0.3103448275862069,
|
|
"acc_stderr": 0.04988718850038745,
|
|
"em_relaxed": 0.3103448275862069,
|
|
"em_relaxed_stderr": 0.04988718850038745
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:chemia|0": {
|
|
"acc": 0.23214285714285715,
|
|
"acc_stderr": 0.05692939024000109,
|
|
"em_relaxed": 0.23214285714285715,
|
|
"em_relaxed_stderr": 0.05692939024000109
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:chemia|0": {
|
|
"acc": 0.21428571428571427,
|
|
"acc_stderr": 0.055328333517248834,
|
|
"em_relaxed": 0.21428571428571427,
|
|
"em_relaxed_stderr": 0.055328333517248834
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"acc": 0.17616580310880828,
|
|
"acc_stderr": 0.027493504244548047,
|
|
"em_relaxed": 0.17616580310880828,
|
|
"em_relaxed_stderr": 0.027493504244548047
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:matematika|0": {
|
|
"acc": 0.20512820512820512,
|
|
"acc_stderr": 0.06550424345215436,
|
|
"em_relaxed": 0.20512820512820512,
|
|
"em_relaxed_stderr": 0.06550424345215436
|
|
},
|
|
"e_tests.prompt3_qa_sk:obcianska-nauka|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.020408163265306124,
|
|
"rouge1_stderr": 0.020408163265306124,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.020408163265306124,
|
|
"rougeL_stderr": 0.020408163265306124
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:dejepis|0": {
|
|
"acc": 0.14583333333333334,
|
|
"acc_stderr": 0.05148145136372286,
|
|
"em_relaxed": 0.14583333333333334,
|
|
"em_relaxed_stderr": 0.05148145136372286
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": {
|
|
"acc": 0.022727272727272728,
|
|
"acc_stderr": 0.022727272727272728
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
|
|
"acc": 0.25704526046114434,
|
|
"acc_stderr": 0.012775945430940199,
|
|
"em_relaxed": 0.2681468830059778,
|
|
"em_relaxed_stderr": 0.012951062793076921
|
|
},
|
|
"sklep.prompt2_nli_3_classes_sk:nli|0": {
|
|
"acc": 0.04936051159072742,
|
|
"acc_stderr": 0.003062545148658699,
|
|
"em_norm_relaxed": 0.06474820143884892,
|
|
"em_norm_relaxed_stderr": 0.003479067694461015
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"acc": 0.2088607594936709,
|
|
"acc_stderr": 0.03244180901572427,
|
|
"em_relaxed": 0.2088607594936709,
|
|
"em_relaxed_stderr": 0.03244180901572427
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"acc": 0.2332089552238806,
|
|
"acc_stderr": 0.01828244336455249,
|
|
"em_relaxed": 0.23507462686567165,
|
|
"em_relaxed_stderr": 0.01833308365747139
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
|
|
"acc": 0.26605504587155965,
|
|
"acc_stderr": 0.042521210223474465,
|
|
"em_relaxed": 0.27522935779816515,
|
|
"em_relaxed_stderr": 0.04297697040536756
|
|
},
|
|
"senti_sk.prompt2_sentiment_3_class_sk|0": {
|
|
"acc": 0.3158007269617276,
|
|
"acc_stderr": 0.006797678423770271,
|
|
"em_norm_relaxed": 0.32050459696386574,
|
|
"em_norm_relaxed_stderr": 0.006824536198921628
|
|
},
|
|
"slovak_tests.prompt1_qa_sk:monitor9|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
|
|
"acc": 0.20833333333333334,
|
|
"acc_stderr": 0.02504221193381759,
|
|
"em_relaxed": 0.20833333333333334,
|
|
"em_relaxed_stderr": 0.02504221193381759
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
|
|
"acc": 0.24107142857142858,
|
|
"acc_stderr": 0.04059867246952688,
|
|
"em_relaxed": 0.25892857142857145,
|
|
"em_relaxed_stderr": 0.041577515398656284
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
|
|
"acc": 0.24647887323943662,
|
|
"acc_stderr": 0.03629342967383697,
|
|
"em_relaxed": 0.24647887323943662,
|
|
"em_relaxed_stderr": 0.03629342967383697
|
|
},
|
|
"e_tests.prompt3_qa_sk:biologia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.019047619047619046,
|
|
"qf1_stderr": 0.019047619047619035,
|
|
"rouge1": 0.014285714285714285,
|
|
"rouge1_stderr": 0.014285714285714287,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.014285714285714285,
|
|
"rougeL_stderr": 0.014285714285714287
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
|
|
"acc": 0.25705329153605017,
|
|
"acc_stderr": 0.02450624980268527,
|
|
"em_relaxed": 0.26332288401253917,
|
|
"em_relaxed_stderr": 0.024698429072332967
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:fyzika|0": {
|
|
"acc": 0.18181818181818182,
|
|
"acc_stderr": 0.08416546361568647,
|
|
"em_relaxed": 0.18181818181818182,
|
|
"em_relaxed_stderr": 0.08416546361568647
|
|
},
|
|
"senti_sk.prompt1_sentiment_3_class_sk|0": {
|
|
"acc": 0.25486422920675644,
|
|
"acc_stderr": 0.006372867628034562,
|
|
"em_norm_relaxed": 0.2572161642078255,
|
|
"em_norm_relaxed_stderr": 0.00639209321447505
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
|
|
"acc": 0.3,
|
|
"acc_stderr": 0.046056618647183814,
|
|
"em_relaxed": 0.3,
|
|
"em_relaxed_stderr": 0.046056618647183814
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
|
|
"acc": 0.16666666666666666,
|
|
"acc_stderr": 0.03708284662416542,
|
|
"em_relaxed": 0.16666666666666666,
|
|
"em_relaxed_stderr": 0.03708284662416542
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"acc": 0.25806451612903225,
|
|
"acc_stderr": 0.029772845386803175,
|
|
"em_relaxed": 0.25806451612903225,
|
|
"em_relaxed_stderr": 0.029772845386803175
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:bio|0": {
|
|
"acc": 0.23008849557522124,
|
|
"acc_stderr": 0.02289335836732507,
|
|
"em_relaxed": 0.2448377581120944,
|
|
"em_relaxed_stderr": 0.02338842519033371
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"acc": 0.18691588785046728,
|
|
"acc_stderr": 0.02671167024582658,
|
|
"em_relaxed": 0.19158878504672897,
|
|
"em_relaxed_stderr": 0.026965681762486853
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
|
|
"acc": 0.2711864406779661,
|
|
"acc_stderr": 0.041100705493392085,
|
|
"em_relaxed": 0.2711864406779661,
|
|
"em_relaxed_stderr": 0.041100705493392085
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.18840579710144928,
|
|
"acc_stderr": 0.047420064740574225,
|
|
"em_relaxed": 0.18840579710144928,
|
|
"em_relaxed_stderr": 0.047420064740574225
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
|
|
"acc": 0.24092409240924093,
|
|
"acc_stderr": 0.0246081660183802,
|
|
"em_relaxed": 0.24752475247524752,
|
|
"em_relaxed_stderr": 0.0248343018351657
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"acc": 0.2727272727272727,
|
|
"acc_stderr": 0.03173071239071724,
|
|
"em_relaxed": 0.2727272727272727,
|
|
"em_relaxed_stderr": 0.03173071239071724
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"acc": 0.23255813953488372,
|
|
"acc_stderr": 0.021502792400356758,
|
|
"em_relaxed": 0.23255813953488372,
|
|
"em_relaxed_stderr": 0.021502792400356758
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:che|0": {
|
|
"acc": 0.27235772357723576,
|
|
"acc_stderr": 0.028441059280558723,
|
|
"em_relaxed": 0.27235772357723576,
|
|
"em_relaxed_stderr": 0.028441059280558723
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
|
|
"acc": 0.18604651162790697,
|
|
"acc_stderr": 0.029758600618213775,
|
|
"em_relaxed": 0.18604651162790697,
|
|
"em_relaxed_stderr": 0.029758600618213775
|
|
},
|
|
"slovak_tests.prompt3_qa_sk:monitor9|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"acc": 0.21481481481481482,
|
|
"acc_stderr": 0.03547854198560827,
|
|
"em_relaxed": 0.21481481481481482,
|
|
"em_relaxed_stderr": 0.03547854198560827
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
|
|
"acc": 0.19696969696969696,
|
|
"acc_stderr": 0.028335609732463348,
|
|
"em_relaxed": 0.19696969696969696,
|
|
"em_relaxed_stderr": 0.028335609732463348
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:informatika|0": {
|
|
"acc": 0.24,
|
|
"acc_stderr": 0.03835319728439192,
|
|
"em_relaxed": 0.24,
|
|
"em_relaxed_stderr": 0.03835319728439192
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"acc": 0.28085106382978725,
|
|
"acc_stderr": 0.029379170464124825,
|
|
"em_relaxed": 0.28085106382978725,
|
|
"em_relaxed_stderr": 0.029379170464124825
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
|
|
"acc": 0.23102310231023102,
|
|
"acc_stderr": 0.02425386079112883,
|
|
"em_relaxed": 0.23102310231023102,
|
|
"em_relaxed_stderr": 0.02425386079112883
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
|
|
"acc": 0.22695035460992907,
|
|
"acc_stderr": 0.024987106365642973,
|
|
"em_relaxed": 0.23049645390070922,
|
|
"em_relaxed_stderr": 0.025123739226872405
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"acc": 0.23333333333333334,
|
|
"acc_stderr": 0.03161296043612007,
|
|
"em_relaxed": 0.23333333333333334,
|
|
"em_relaxed_stderr": 0.03161296043612007
|
|
},
|
|
"sklep.prompt1_nli_binary_sk:rte|0": {
|
|
"acc": 0.26385542168674697,
|
|
"acc_stderr": 0.010820351700856653,
|
|
"em_norm_relaxed": 0.26626506024096386,
|
|
"em_norm_relaxed_stderr": 0.01085184289579353
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": {
|
|
"acc": 0.22959183673469388,
|
|
"acc_stderr": 0.02126915716826347,
|
|
"em_relaxed": 0.22959183673469388,
|
|
"em_relaxed_stderr": 0.02126915716826347
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
|
|
"acc": 0.3486238532110092,
|
|
"acc_stderr": 0.04585456701913343,
|
|
"em_relaxed": 0.3486238532110092,
|
|
"em_relaxed_stderr": 0.04585456701913343
|
|
},
|
|
"slohry.prompt1_inflection_sk:inflection_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_declension_sk:declension_high|0": {
|
|
"acc": 0.0748502994011976,
|
|
"acc_stderr": 0.014420511342862789,
|
|
"em_norm_relaxed": 0.0748502994011976,
|
|
"em_norm_relaxed_stderr": 0.014420511342862789
|
|
},
|
|
"slohry.prompt3_gradation_sk:gradation_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_inflection_sk:inflection_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_gradation_sk:gradation_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_declension_sk:declension_high|0": {
|
|
"acc": 0.059880239520958084,
|
|
"acc_stderr": 0.013002032231530652,
|
|
"em_norm_relaxed": 0.059880239520958084,
|
|
"em_norm_relaxed_stderr": 0.013002032231530652
|
|
},
|
|
"chiby.prompt2_chiby_sk|0": {
|
|
"acc": 0.025349650349650348,
|
|
"acc_stderr": 0.0026834904197041206,
|
|
"em_norm_relaxed": 0.02564102564102564,
|
|
"em_norm_relaxed_stderr": 0.002698465265697882
|
|
},
|
|
"slovak_trivia.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.28611898016997167,
|
|
"acc_stderr": 0.024088786371538086,
|
|
"em_relaxed": 0.28611898016997167,
|
|
"em_relaxed_stderr": 0.024088786371538086
|
|
},
|
|
"slohry.prompt1_inflection_sk:inflection_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_gradation_sk:gradation_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"chiby.prompt3_chiby_sk|0": {
|
|
"acc": 0.1622960372960373,
|
|
"acc_stderr": 0.00629490070363013,
|
|
"em_norm_relaxed": 0.16258741258741258,
|
|
"em_norm_relaxed_stderr": 0.00629945304550759
|
|
},
|
|
"slohry.prompt2_declension_sk:declension_mid|0": {
|
|
"acc": 0.05389221556886228,
|
|
"acc_stderr": 0.012374031301903447,
|
|
"em_norm_relaxed": 0.05389221556886228,
|
|
"em_norm_relaxed_stderr": 0.012374031301903447
|
|
},
|
|
"slohry.prompt2_gradation_sk:gradation_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_gradation_sk:gradation_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_identity_form_sk:identity_form_low|0": {
|
|
"acc": 0.005988023952095809,
|
|
"acc_stderr": 0.004227809946139441,
|
|
"em_norm_relaxed": 0.005988023952095809,
|
|
"em_norm_relaxed_stderr": 0.004227809946139441
|
|
},
|
|
"slohry.prompt3_declension_sk:declension_high|0": {
|
|
"acc": 0.0718562874251497,
|
|
"acc_stderr": 0.014152002095505148,
|
|
"em_norm_relaxed": 0.0718562874251497,
|
|
"em_norm_relaxed_stderr": 0.014152002095505148
|
|
},
|
|
"slohry.prompt3_declension_sk:declension_mid|0": {
|
|
"acc": 0.0658682634730539,
|
|
"acc_stderr": 0.013593148263003514,
|
|
"em_norm_relaxed": 0.0658682634730539,
|
|
"em_norm_relaxed_stderr": 0.013593148263003514
|
|
},
|
|
"slohry.prompt1_identity_form_sk:identity_form_high|0": {
|
|
"acc": 0.0029940119760479044,
|
|
"acc_stderr": 0.0029940119760478922,
|
|
"em_norm_relaxed": 0.0029940119760479044,
|
|
"em_norm_relaxed_stderr": 0.0029940119760478922
|
|
},
|
|
"slohry.prompt3_declension_sk:declension_low|0": {
|
|
"acc": 0.04790419161676647,
|
|
"acc_stderr": 0.011703209185183546,
|
|
"em_norm_relaxed": 0.04790419161676647,
|
|
"em_norm_relaxed_stderr": 0.011703209185183546
|
|
},
|
|
"slohry.prompt1_declension_sk:declension_mid|0": {
|
|
"acc": 0.0718562874251497,
|
|
"acc_stderr": 0.014152002095505158,
|
|
"em_norm_relaxed": 0.0718562874251497,
|
|
"em_norm_relaxed_stderr": 0.014152002095505158
|
|
},
|
|
"slohry.prompt3_inflection_sk:inflection_high|0": {
|
|
"acc": 0.008982035928143712,
|
|
"acc_stderr": 0.005170184492247012,
|
|
"em_norm_relaxed": 0.008982035928143712,
|
|
"em_norm_relaxed_stderr": 0.005170184492247012
|
|
},
|
|
"slohry.prompt2_declension_sk:declension_low|0": {
|
|
"acc": 0.029940119760479042,
|
|
"acc_stderr": 0.00933907625586894,
|
|
"em_norm_relaxed": 0.029940119760479042,
|
|
"em_norm_relaxed_stderr": 0.00933907625586894
|
|
},
|
|
"slohry.prompt3_gradation_sk:gradation_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_identity_form_sk:identity_form_low|0": {
|
|
"acc": 0.008982035928143712,
|
|
"acc_stderr": 0.005170184492247015,
|
|
"em_norm_relaxed": 0.008982035928143712,
|
|
"em_norm_relaxed_stderr": 0.005170184492247015
|
|
},
|
|
"slovak_trivia.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.26628895184135976,
|
|
"acc_stderr": 0.02355959235547647,
|
|
"em_relaxed": 0.26628895184135976,
|
|
"em_relaxed_stderr": 0.02355959235547647
|
|
},
|
|
"analogies_sk.prompt3_analogy_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_gradation_sk:gradation_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_inflection_sk:inflection_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_inflection_sk:inflection_high|0": {
|
|
"acc": 0.0029940119760479044,
|
|
"acc_stderr": 0.0029940119760478966,
|
|
"em_norm_relaxed": 0.0029940119760479044,
|
|
"em_norm_relaxed_stderr": 0.0029940119760478966
|
|
},
|
|
"slohry.prompt3_inflection_sk:inflection_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_identity_form_sk:identity_form_high|0": {
|
|
"acc": 0.005988023952095809,
|
|
"acc_stderr": 0.004227809946139438,
|
|
"em_norm_relaxed": 0.005988023952095809,
|
|
"em_norm_relaxed_stderr": 0.004227809946139438
|
|
},
|
|
"slohry.prompt2_identity_form_sk:identity_form_mid|0": {
|
|
"acc": 0.02694610778443114,
|
|
"acc_stderr": 0.008873487700057157,
|
|
"em_norm_relaxed": 0.02694610778443114,
|
|
"em_norm_relaxed_stderr": 0.008873487700057157
|
|
},
|
|
"slohry.prompt3_identity_form_sk:identity_form_mid|0": {
|
|
"acc": 0.008982035928143712,
|
|
"acc_stderr": 0.0051701844922470075,
|
|
"em_norm_relaxed": 0.008982035928143712,
|
|
"em_norm_relaxed_stderr": 0.0051701844922470075
|
|
},
|
|
"slohry.prompt3_identity_form_sk:identity_form_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_declension_sk:declension_low|0": {
|
|
"acc": 0.05089820359281437,
|
|
"acc_stderr": 0.012044409374838807,
|
|
"em_norm_relaxed": 0.05089820359281437,
|
|
"em_norm_relaxed_stderr": 0.012044409374838807
|
|
},
|
|
"slohry.prompt1_gradation_sk:gradation_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"analogies_sk.prompt2_analogy_sk|0": {
|
|
"acc": 0.0007544322897019992,
|
|
"acc_stderr": 0.0005333635248765946
|
|
},
|
|
"analogies_sk.prompt1_analogy_sk|0": {
|
|
"acc": 0.001131648434552999,
|
|
"acc_stderr": 0.0006531109317700804
|
|
},
|
|
"slohry.prompt1_gradation_sk:gradation_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_identity_form_sk:identity_form_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_inflection_sk:inflection_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_inflection_sk:inflection_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"chiby.prompt1_chiby_sk|0": {
|
|
"acc": 0.23572261072261072,
|
|
"acc_stderr": 0.00724629095400738,
|
|
"em_norm_relaxed": 0.23892773892773891,
|
|
"em_norm_relaxed_stderr": 0.007280075348854293
|
|
},
|
|
"slovak_trivia.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.28328611898017,
|
|
"acc_stderr": 0.024016749232030024,
|
|
"em_relaxed": 0.28328611898017,
|
|
"em_relaxed_stderr": 0.024016749232030024
|
|
},
|
|
"slohry.prompt1_identity_form_sk:identity_form_mid|0": {
|
|
"acc": 0.011976047904191617,
|
|
"acc_stderr": 0.005960989845848331,
|
|
"em_norm_relaxed": 0.011976047904191617,
|
|
"em_norm_relaxed_stderr": 0.005960989845848331
|
|
},
|
|
"movie_review_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.14890438247011953,
|
|
"acc": 0.1457503320053121,
|
|
"em_norm_relaxed_stderr": 0.007823268108665498,
|
|
"acc_stderr": 0.007746689156547597,
|
|
"em_norm_relaxed__prompts": {
|
|
"1": 0.10109561752988047,
|
|
"3": 0.20916334661354583,
|
|
"2": 0.13645418326693226
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.09860557768924302,
|
|
"3": 0.20766932270916336,
|
|
"2": 0.13097609561752988
|
|
}
|
|
},
|
|
"movie_review_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.14890438247011953,
|
|
"em_norm_relaxed_stderr": 0.007823268108665498,
|
|
"acc": 0.1457503320053121,
|
|
"acc_stderr": 0.007746689156547597
|
|
},
|
|
"e_tests_mqa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.24309199020636585,
|
|
"acc": 0.18150087260034903,
|
|
"em_relaxed_stderr": 0.03621678600189384,
|
|
"acc_stderr": 0.03214832055891622,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.24239244491080797,
|
|
"1": 0.2497376705141658,
|
|
"3": 0.23714585519412382
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.18175018698578907,
|
|
"1": 0.1862378459237098,
|
|
"3": 0.17651458489154825
|
|
}
|
|
},
|
|
"e_tests_mqa_AVERAGE|0": {
|
|
"em_relaxed": 0.24309199020636585,
|
|
"em_relaxed_stderr": 0.03621678600189384,
|
|
"acc": 0.18150087260034903,
|
|
"acc_stderr": 0.03214832055891622
|
|
},
|
|
"mmlu_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.24332669831062115,
|
|
"acc": 0.24148930740570612,
|
|
"em_relaxed_stderr": 0.0266013460946822,
|
|
"acc_stderr": 0.026529494985304103,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.24628693921298422,
|
|
"3": 0.2474353085285561,
|
|
"1": 0.23625784719032308
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.2440667585362119,
|
|
"3": 0.24437299035369775,
|
|
"1": 0.23602817332720868
|
|
}
|
|
},
|
|
"mmlu_AVERAGE|0": {
|
|
"em_relaxed": 0.24332669831062115,
|
|
"em_relaxed_stderr": 0.0266013460946822,
|
|
"acc": 0.24148930740570612,
|
|
"acc_stderr": 0.026529494985304103
|
|
},
|
|
"belebele_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.267037037037037,
|
|
"acc": 0.26666666666666666,
|
|
"em_relaxed_stderr": 0.014753176443555857,
|
|
"acc_stderr": 0.014746835660938531,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.2722222222222222,
|
|
"1": 0.2577777777777778,
|
|
"2": 0.27111111111111114
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.27111111111111114,
|
|
"1": 0.2577777777777778,
|
|
"2": 0.27111111111111114
|
|
}
|
|
},
|
|
"belebele_AVERAGE|0": {
|
|
"em_relaxed": 0.267037037037037,
|
|
"em_relaxed_stderr": 0.014753176443555857,
|
|
"acc": 0.26666666666666666,
|
|
"acc_stderr": 0.014746835660938531
|
|
},
|
|
"senti_sk_binary_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.49356223175965663,
|
|
"acc": 0.4873927038626609,
|
|
"em_norm_relaxed_stderr": 0.007613926970563059,
|
|
"acc_stderr": 0.0076136774621452415,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.3261802575107296,
|
|
"1": 0.4069206008583691,
|
|
"3": 0.7475858369098712
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.32135193133047213,
|
|
"1": 0.39914163090128757,
|
|
"3": 0.7416845493562232
|
|
}
|
|
},
|
|
"senti_sk_binary_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.49356223175965663,
|
|
"em_norm_relaxed_stderr": 0.007613926970563059,
|
|
"acc": 0.4873927038626609,
|
|
"acc_stderr": 0.0076136774621452415
|
|
},
|
|
"slovak_tests_mqa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.2600800246229609,
|
|
"acc": 0.2600800246229609,
|
|
"em_relaxed_stderr": 0.02573346982142187,
|
|
"acc_stderr": 0.02573346982142187,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.26685133887349954,
|
|
"3": 0.25761772853185594,
|
|
"1": 0.25577100646352724
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.26685133887349954,
|
|
"3": 0.25761772853185594,
|
|
"1": 0.25577100646352724
|
|
}
|
|
},
|
|
"slovak_tests_mqa_AVERAGE|0": {
|
|
"em_relaxed": 0.2600800246229609,
|
|
"em_relaxed_stderr": 0.02573346982142187,
|
|
"acc": 0.2600800246229609,
|
|
"acc_stderr": 0.02573346982142187
|
|
},
|
|
"e_tests_qa_SK_AVERAGE|0": {
|
|
"rouge2": 0.003228194982976972,
|
|
"rouge1": 0.014818769352489746,
|
|
"rougeL": 0.014818769352489746,
|
|
"qf1": 0.007878222192390297,
|
|
"qem": 0.012039127163280662,
|
|
"rouge2_stderr": 0.002547306056000719,
|
|
"rouge1_stderr": 0.008024257279824093,
|
|
"rougeL_stderr": 0.008024257279824093,
|
|
"qf1_stderr": 0.005529975861330393,
|
|
"qem_stderr": 0.00874750521016363,
|
|
"rouge2__prompts": {
|
|
"3": 0.0030234626171420755,
|
|
"2": 0.003310759969902182,
|
|
"1": 0.0033503623618866578
|
|
},
|
|
"rouge1__prompts": {
|
|
"3": 0.013608596069092682,
|
|
"2": 0.014902802239145354,
|
|
"1": 0.015944909749231198
|
|
},
|
|
"rougeL__prompts": {
|
|
"3": 0.013608596069092682,
|
|
"2": 0.014902802239145354,
|
|
"1": 0.015944909749231198
|
|
},
|
|
"qf1__prompts": {
|
|
"3": 0.005311379630859115,
|
|
"2": 0.008663871869289476,
|
|
"1": 0.009659415077022301
|
|
},
|
|
"qem__prompts": {
|
|
"3": 0.011286681715575621,
|
|
"2": 0.013544018058690745,
|
|
"1": 0.011286681715575621
|
|
}
|
|
},
|
|
"e_tests_qa_AVERAGE|0": {
|
|
"rouge2": 0.003228194982976972,
|
|
"rouge2_stderr": 0.002547306056000719,
|
|
"rouge1": 0.014818769352489746,
|
|
"rouge1_stderr": 0.008024257279824093,
|
|
"rougeL": 0.014818769352489746,
|
|
"rougeL_stderr": 0.008024257279824093,
|
|
"qf1": 0.007878222192390297,
|
|
"qf1_stderr": 0.005529975861330393,
|
|
"qem": 0.012039127163280662,
|
|
"qem_stderr": 0.00874750521016363
|
|
},
|
|
"scio_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.25434173669467786,
|
|
"acc": 0.250046685340803,
|
|
"em_relaxed_stderr": 0.02074943266589043,
|
|
"acc_stderr": 0.020633005498944187,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.27675070028011206,
|
|
"2": 0.2549019607843137,
|
|
"1": 0.23137254901960785
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.2700280112044818,
|
|
"2": 0.24929971988795518,
|
|
"1": 0.230812324929972
|
|
}
|
|
},
|
|
"scio_AVERAGE|0": {
|
|
"em_relaxed": 0.25434173669467786,
|
|
"em_relaxed_stderr": 0.02074943266589043,
|
|
"acc": 0.250046685340803,
|
|
"acc_stderr": 0.020633005498944187
|
|
},
|
|
"arc_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.24664955802680352,
|
|
"acc": 0.2446535500427716,
|
|
"em_relaxed_stderr": 0.012612937670503793,
|
|
"acc_stderr": 0.012578166445440875,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.24721984602224123,
|
|
"3": 0.24550898203592814,
|
|
"1": 0.24721984602224123
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.24123182207014543,
|
|
"3": 0.24550898203592814,
|
|
"1": 0.24721984602224123
|
|
}
|
|
},
|
|
"arc_AVERAGE|0": {
|
|
"em_relaxed": 0.24664955802680352,
|
|
"em_relaxed_stderr": 0.012612937670503793,
|
|
"acc": 0.2446535500427716,
|
|
"acc_stderr": 0.012578166445440875
|
|
},
|
|
"sklep_rte_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.31506024096385543,
|
|
"acc": 0.3140562248995984,
|
|
"em_norm_relaxed_stderr": 0.010750305246379226,
|
|
"acc_stderr": 0.010735367420948657,
|
|
"em_norm_relaxed__prompts": {
|
|
"3": 0.16566265060240964,
|
|
"2": 0.5132530120481927,
|
|
"1": 0.26626506024096386
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.1650602409638554,
|
|
"2": 0.5132530120481927,
|
|
"1": 0.26385542168674697
|
|
}
|
|
},
|
|
"sklep_rte_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.31506024096385543,
|
|
"em_norm_relaxed_stderr": 0.010750305246379226,
|
|
"acc": 0.3140562248995984,
|
|
"acc_stderr": 0.010735367420948657
|
|
},
|
|
"driving_test_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.2710413694721826,
|
|
"acc": 0.2710413694721826,
|
|
"em_relaxed_stderr": 0.016450865339756114,
|
|
"acc_stderr": 0.016450865339756114,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.17261055634807418,
|
|
"1": 0.3666191155492154,
|
|
"2": 0.2738944365192582
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.17261055634807418,
|
|
"1": 0.3666191155492154,
|
|
"2": 0.2738944365192582
|
|
}
|
|
},
|
|
"driving_test_AVERAGE|0": {
|
|
"em_relaxed": 0.2710413694721826,
|
|
"em_relaxed_stderr": 0.016450865339756114,
|
|
"acc": 0.2710413694721826,
|
|
"acc_stderr": 0.016450865339756114
|
|
},
|
|
"sklep_sentiment_analysis_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.11388355726167626,
|
|
"acc": 0.10748560460652591,
|
|
"em_norm_relaxed_stderr": 0.009453015726623323,
|
|
"acc_stderr": 0.009172256989852929,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.19193857965451055,
|
|
"3": 0.09404990403071017,
|
|
"1": 0.05566218809980806
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.1833013435700576,
|
|
"3": 0.09213051823416507,
|
|
"1": 0.04702495201535509
|
|
}
|
|
},
|
|
"sklep_sentiment_analysis_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.11388355726167626,
|
|
"em_norm_relaxed_stderr": 0.009453015726623323,
|
|
"acc": 0.10748560460652591,
|
|
"acc_stderr": 0.009172256989852929
|
|
},
|
|
"slovakcopa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.27066666666666667,
|
|
"acc": 0.27,
|
|
"em_relaxed_stderr": 0.016480088564366616,
|
|
"acc_stderr": 0.016473550449007283,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.402,
|
|
"2": 0.394,
|
|
"3": 0.016
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.402,
|
|
"2": 0.392,
|
|
"3": 0.016
|
|
}
|
|
},
|
|
"slovakcopa_AVERAGE|0": {
|
|
"em_relaxed": 0.27066666666666667,
|
|
"em_relaxed_stderr": 0.016480088564366616,
|
|
"acc": 0.27,
|
|
"acc_stderr": 0.016473550449007283
|
|
},
|
|
"sklep_qa_SK_AVERAGE|0": {
|
|
"qf1": 0.14283353603077906,
|
|
"qem": 0.05203659257713312,
|
|
"qf1_stderr": 0.0027203043184777205,
|
|
"qem_stderr": 0.002268897741053689,
|
|
"qf1__prompts": {
|
|
"2": 0.14428713100967,
|
|
"1": 0.1424834912487429,
|
|
"3": 0.1417299858339243
|
|
},
|
|
"qem__prompts": {
|
|
"2": 0.05238443076280914,
|
|
"1": 0.05248878221851195,
|
|
"3": 0.05123656475007826
|
|
}
|
|
},
|
|
"sklep_qa_AVERAGE|0": {
|
|
"qf1": 0.14283353603077906,
|
|
"qf1_stderr": 0.0027203043184777205,
|
|
"qem": 0.05203659257713312,
|
|
"qem_stderr": 0.002268897741053689
|
|
},
|
|
"senti_sk_3class_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.38621623547858314,
|
|
"acc": 0.3813698239612287,
|
|
"em_norm_relaxed_stderr": 0.006810719788072698,
|
|
"acc_stderr": 0.006801058501709849,
|
|
"em_norm_relaxed__prompts": {
|
|
"3": 0.5809279452640581,
|
|
"2": 0.32050459696386574,
|
|
"1": 0.2572161642078255
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.573444515715202,
|
|
"2": 0.3158007269617276,
|
|
"1": 0.25486422920675644
|
|
}
|
|
},
|
|
"senti_sk_3class_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.38621623547858314,
|
|
"em_norm_relaxed_stderr": 0.006810719788072698,
|
|
"acc": 0.3813698239612287,
|
|
"acc_stderr": 0.006801058501709849
|
|
},
|
|
"multi_wiki_qa_SK_AVERAGE|0": {
|
|
"qf1": 0.12354806704071784,
|
|
"qem": 0.07884730838503101,
|
|
"qf1_stderr": 0.003950374051156623,
|
|
"qem_stderr": 0.0038122173887514235,
|
|
"qf1__prompts": {
|
|
"3": 0.12171966052606027,
|
|
"2": 0.12697709233547141,
|
|
"1": 0.12194744826062183
|
|
},
|
|
"qem__prompts": {
|
|
"3": 0.07884730838503101,
|
|
"2": 0.08204922953772263,
|
|
"1": 0.0756453872323394
|
|
}
|
|
},
|
|
"multi_wiki_qa_AVERAGE|0": {
|
|
"qf1": 0.12354806704071784,
|
|
"qf1_stderr": 0.003950374051156623,
|
|
"qem": 0.07884730838503101,
|
|
"qem_stderr": 0.0038122173887514235
|
|
},
|
|
"ifeval_sk_SK_AVERAGE|0": {
|
|
"prompt_level_strict_acc": 0.10242085661080075,
|
|
"prompt_level_loose_acc": 0.10242085661080075,
|
|
"inst_level_strict_acc": 0.21307506053268765,
|
|
"inst_level_loose_acc": 0.21307506053268765,
|
|
"prompt_level_strict_acc_stderr": 0.01309628698247887,
|
|
"prompt_level_loose_acc_stderr": 0.01309628698247887,
|
|
"inst_level_strict_acc_stderr": 0.00041218013349389904,
|
|
"inst_level_loose_acc_stderr": 0.00041218013349389904
|
|
},
|
|
"ifeval_sk_AVERAGE|0": {
|
|
"prompt_level_strict_acc": 0.10242085661080075,
|
|
"prompt_level_strict_acc_stderr": 0.01309628698247887,
|
|
"prompt_level_loose_acc": 0.10242085661080075,
|
|
"prompt_level_loose_acc_stderr": 0.01309628698247887,
|
|
"inst_level_strict_acc": 0.21307506053268765,
|
|
"inst_level_strict_acc_stderr": 0.00041218013349389904,
|
|
"inst_level_loose_acc": 0.21307506053268765,
|
|
"inst_level_loose_acc_stderr": 0.00041218013349389904
|
|
},
|
|
"globalmmlu_lite_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.2783333333333333,
|
|
"acc": 0.2758333333333333,
|
|
"em_relaxed_stderr": 0.022429986391836288,
|
|
"acc_stderr": 0.022369608398602674,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.2925,
|
|
"1": 0.2725,
|
|
"2": 0.27
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.2875,
|
|
"1": 0.2725,
|
|
"2": 0.2675
|
|
}
|
|
},
|
|
"globalmmlu_lite_AVERAGE|0": {
|
|
"em_relaxed": 0.2783333333333333,
|
|
"em_relaxed_stderr": 0.022429986391836288,
|
|
"acc": 0.2758333333333333,
|
|
"acc_stderr": 0.022369608398602674
|
|
},
|
|
"sklep_nli_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.024047428723687717,
|
|
"acc": 0.0187183586464162,
|
|
"em_norm_relaxed_stderr": 0.0017224579545798097,
|
|
"acc_stderr": 0.001563138066747669,
|
|
"em_norm_relaxed__prompts": {
|
|
"3": 0.002398081534772182,
|
|
"1": 0.004996003197442046,
|
|
"2": 0.06474820143884892
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.002398081534772182,
|
|
"1": 0.004396482813749001,
|
|
"2": 0.04936051159072742
|
|
}
|
|
},
|
|
"sklep_nli_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.024047428723687717,
|
|
"em_norm_relaxed_stderr": 0.0017224579545798097,
|
|
"acc": 0.0187183586464162,
|
|
"acc_stderr": 0.001563138066747669
|
|
},
|
|
"slovak_tests_qa_SK_AVERAGE|0": {
|
|
"rouge2": 0.0030834556044640076,
|
|
"rouge1": 0.0149009568631566,
|
|
"rougeL": 0.0149009568631566,
|
|
"qf1": 0.008916490572837322,
|
|
"qem": 0.00980392156862745,
|
|
"rouge2_stderr": 0.00228223362995418,
|
|
"rouge1_stderr": 0.00523977237139308,
|
|
"rougeL_stderr": 0.00523977237139308,
|
|
"qf1_stderr": 0.005125765723300312,
|
|
"qem_stderr": 0.006367605535927983,
|
|
"rouge2__prompts": {
|
|
"1": 0.003274643190609577,
|
|
"2": 0.0021008403361344537,
|
|
"3": 0.0038748832866479918
|
|
},
|
|
"rouge1__prompts": {
|
|
"1": 0.014745243264740951,
|
|
"2": 0.015252537196116269,
|
|
"3": 0.014705090128612585
|
|
},
|
|
"rougeL__prompts": {
|
|
"1": 0.014745243264740951,
|
|
"2": 0.015252537196116269,
|
|
"3": 0.014705090128612585
|
|
},
|
|
"qf1__prompts": {
|
|
"1": 0.00765639589169001,
|
|
"2": 0.006349206349206349,
|
|
"3": 0.012743869477615606
|
|
},
|
|
"qem__prompts": {
|
|
"1": 0.008403361344537815,
|
|
"2": 0.008403361344537815,
|
|
"3": 0.012605042016806723
|
|
}
|
|
},
|
|
"slovak_tests_qa_AVERAGE|0": {
|
|
"rouge2": 0.0030834556044640076,
|
|
"rouge2_stderr": 0.00228223362995418,
|
|
"rouge1": 0.0149009568631566,
|
|
"rouge1_stderr": 0.00523977237139308,
|
|
"rougeL": 0.0149009568631566,
|
|
"rougeL_stderr": 0.00523977237139308,
|
|
"qf1": 0.008916490572837322,
|
|
"qf1_stderr": 0.005125765723300312,
|
|
"qem": 0.00980392156862745,
|
|
"qem_stderr": 0.006367605535927983
|
|
},
|
|
"skj1_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.0033333333333333335,
|
|
"acc": 0.0,
|
|
"em_relaxed_stderr": 0.0033333333333333322,
|
|
"acc_stderr": 0.0,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.01,
|
|
"3": 0.0,
|
|
"2": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.0,
|
|
"3": 0.0,
|
|
"2": 0.0
|
|
}
|
|
},
|
|
"skj1_AVERAGE|0": {
|
|
"em_relaxed": 0.0033333333333333335,
|
|
"em_relaxed_stderr": 0.0033333333333333322,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"financial_exams_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.18640679660169915,
|
|
"acc": 0.18615692153923039,
|
|
"em_relaxed_stderr": 0.010665889604449459,
|
|
"acc_stderr": 0.010660482281487506,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.18590704647676162,
|
|
"1": 0.18290854572713644,
|
|
"2": 0.1904047976011994
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.18590704647676162,
|
|
"1": 0.18290854572713644,
|
|
"2": 0.1896551724137931
|
|
}
|
|
},
|
|
"financial_exams_AVERAGE|0": {
|
|
"em_relaxed": 0.18640679660169915,
|
|
"em_relaxed_stderr": 0.010665889604449459,
|
|
"acc": 0.18615692153923039,
|
|
"acc_stderr": 0.010660482281487506
|
|
},
|
|
"truthful_qa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.32390745501285345,
|
|
"acc": 0.31876606683804626,
|
|
"em_relaxed_stderr": 0.01598799102785622,
|
|
"acc_stderr": 0.015861196644299858,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.40359897172236503,
|
|
"3": 0.14524421593830333,
|
|
"1": 0.42287917737789205
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.397172236503856,
|
|
"3": 0.13624678663239073,
|
|
"1": 0.42287917737789205
|
|
}
|
|
},
|
|
"truthful_qa_AVERAGE|0": {
|
|
"em_relaxed": 0.32390745501285345,
|
|
"em_relaxed_stderr": 0.01598799102785622,
|
|
"acc": 0.31876606683804626,
|
|
"acc_stderr": 0.015861196644299858
|
|
},
|
|
"sklep_hate_speech_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.21430376547889815,
|
|
"acc": 0.2137983320697498,
|
|
"em_norm_relaxed_stderr": 0.010710417940514876,
|
|
"acc_stderr": 0.010698565134938456,
|
|
"em_norm_relaxed__prompts": {
|
|
"3": 0.09401061410159212,
|
|
"1": 0.35329795299469297,
|
|
"2": 0.1956027293404094
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.0932524639878696,
|
|
"1": 0.3525398028809704,
|
|
"2": 0.1956027293404094
|
|
}
|
|
},
|
|
"sklep_hate_speech_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.21430376547889815,
|
|
"em_norm_relaxed_stderr": 0.010710417940514876,
|
|
"acc": 0.2137983320697498,
|
|
"acc_stderr": 0.010698565134938456
|
|
},
|
|
"toxic_sk_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.19977298524404086,
|
|
"acc": 0.19901626939084374,
|
|
"em_norm_relaxed_stderr": 0.011151493876554907,
|
|
"acc_stderr": 0.011139685234547891,
|
|
"em_norm_relaxed__prompts": {
|
|
"3": 0.007945516458569807,
|
|
"2": 0.23723041997729852,
|
|
"1": 0.35414301929625425
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.007945516458569807,
|
|
"2": 0.23609534619750283,
|
|
"1": 0.3530079455164586
|
|
}
|
|
},
|
|
"toxic_sk_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.19977298524404086,
|
|
"em_norm_relaxed_stderr": 0.011151493876554907,
|
|
"acc": 0.19901626939084374,
|
|
"acc_stderr": 0.011139685234547891
|
|
},
|
|
"slohry_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.016983016983016984,
|
|
"acc": 0.016983016983016984,
|
|
"em_norm_relaxed_stderr": 0.004436902963904142,
|
|
"acc_stderr": 0.004436902963904142,
|
|
"em_norm_relaxed__prompts": {
|
|
"1": 0.018731268731268732,
|
|
"3": 0.016983016983016984,
|
|
"2": 0.015234765234765234
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.018731268731268732,
|
|
"3": 0.016983016983016984,
|
|
"2": 0.015234765234765234
|
|
}
|
|
},
|
|
"slohry_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.016983016983016984,
|
|
"em_norm_relaxed_stderr": 0.004436902963904142,
|
|
"acc": 0.016983016983016984,
|
|
"acc_stderr": 0.004436902963904142
|
|
},
|
|
"chiby_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.14238539238539238,
|
|
"acc": 0.1411227661227661,
|
|
"em_norm_relaxed_stderr": 0.005425997886686588,
|
|
"acc_stderr": 0.005408227359113877,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.02564102564102564,
|
|
"3": 0.16258741258741258,
|
|
"1": 0.23892773892773891
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.025349650349650348,
|
|
"3": 0.1622960372960373,
|
|
"1": 0.23572261072261072
|
|
}
|
|
},
|
|
"chiby_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.14238539238539238,
|
|
"em_norm_relaxed_stderr": 0.005425997886686588,
|
|
"acc": 0.1411227661227661,
|
|
"acc_stderr": 0.005408227359113877
|
|
},
|
|
"slovak_trivia_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.2785646836638338,
|
|
"acc": 0.2785646836638338,
|
|
"em_relaxed_stderr": 0.023888375986348194,
|
|
"acc_stderr": 0.023888375986348194,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.28611898016997167,
|
|
"1": 0.26628895184135976,
|
|
"2": 0.28328611898017
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.28611898016997167,
|
|
"1": 0.26628895184135976,
|
|
"2": 0.28328611898017
|
|
}
|
|
},
|
|
"slovak_trivia_AVERAGE|0": {
|
|
"em_relaxed": 0.2785646836638338,
|
|
"em_relaxed_stderr": 0.023888375986348194,
|
|
"acc": 0.2785646836638338,
|
|
"acc_stderr": 0.023888375986348194
|
|
},
|
|
"analogies_sk_SK_AVERAGE|0": {
|
|
"acc": 0.000628693574751666,
|
|
"acc_stderr": 0.00039549148554889166,
|
|
"acc__prompts": {
|
|
"3": 0.0,
|
|
"2": 0.0007544322897019992,
|
|
"1": 0.001131648434552999
|
|
}
|
|
},
|
|
"analogies_sk_AVERAGE|0": {
|
|
"acc": 0.000628693574751666,
|
|
"acc_stderr": 0.00039549148554889166
|
|
},
|
|
"SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.20551192367489274,
|
|
"acc": 0.21129673365203147,
|
|
"em_relaxed": 0.24052159099864373,
|
|
"rouge2": 0.00315582529372049,
|
|
"rouge1": 0.014859863107823173,
|
|
"rougeL": 0.014859863107823173,
|
|
"qf1": 0.07079407895918113,
|
|
"qem": 0.038181737423518054,
|
|
"prompt_level_strict_acc": 0.10242085661080075,
|
|
"prompt_level_loose_acc": 0.10242085661080075,
|
|
"inst_level_strict_acc": 0.21307506053268765,
|
|
"inst_level_loose_acc": 0.21307506053268765,
|
|
"em_norm_relaxed_stderr": 0.0025722619842950684,
|
|
"acc_stderr": 0.0031822329617778393,
|
|
"em_relaxed_stderr": 0.0056971140284625835,
|
|
"rouge2_stderr": 0.001710070063230727,
|
|
"rouge1_stderr": 0.0047917616644827035,
|
|
"rougeL_stderr": 0.0047917616644827035,
|
|
"qf1_stderr": 0.002234105218066023,
|
|
"qem_stderr": 0.0029234626947495608,
|
|
"prompt_level_strict_acc_stderr": 0.01309628698247887,
|
|
"prompt_level_loose_acc_stderr": 0.01309628698247887,
|
|
"inst_level_strict_acc_stderr": 0.00041218013349389904,
|
|
"inst_level_loose_acc_stderr": 0.00041218013349389904
|
|
},
|
|
"AVERAGE|0": {
|
|
"em_norm_relaxed": 0.185433666807816,
|
|
"acc": 0.21317053562216026,
|
|
"em_relaxed": 0.2602872791374196,
|
|
"rouge2": 0.00315582529372049,
|
|
"rouge1": 0.014859863107823173,
|
|
"rougeL": 0.014859863107823173,
|
|
"qf1": 0.07079407895918113,
|
|
"qem": 0.038181737423518054,
|
|
"prompt_level_strict_acc": 0.10242085661080075,
|
|
"prompt_level_loose_acc": 0.10242085661080075,
|
|
"inst_level_strict_acc": 0.21307506053268765,
|
|
"inst_level_loose_acc": 0.21307506053268765,
|
|
"em_norm_relaxed_stderr": 0.002756064620435493,
|
|
"acc_stderr": 0.003457735075973522,
|
|
"em_relaxed_stderr": 0.0061656193838727944,
|
|
"rouge2_stderr": 0.001710070063230727,
|
|
"rouge1_stderr": 0.0047917616644827035,
|
|
"rougeL_stderr": 0.0047917616644827035,
|
|
"qf1_stderr": 0.002234105218066023,
|
|
"qem_stderr": 0.0029234626947495608,
|
|
"prompt_level_strict_acc_stderr": 0.01309628698247887,
|
|
"prompt_level_loose_acc_stderr": 0.01309628698247887,
|
|
"inst_level_strict_acc_stderr": 0.00041218013349389904,
|
|
"inst_level_loose_acc_stderr": 0.00041218013349389904
|
|
}
|
|
}
|
|
} |