{ "model_name": "models/ministral-base", "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "max_samples": null, "job_id": "0", "start_time": 883355.287443326, "end_time": 883520.97395618, "total_evaluation_time_secondes": "165.68651285406668", "model_config": { "model_name": "models/ministral-base", "generation_parameters": { "num_blocks": null, "block_size": null, "early_stopping": null, "repetition_penalty": null, "frequency_penalty": null, "length_penalty": null, "presence_penalty": null, "max_new_tokens": null, "min_new_tokens": null, "seed": null, "stop_tokens": null, "temperature": 0, "top_k": null, "min_p": null, "top_p": null, "truncate_prompt": null, "cache_implementation": null, "response_format": null, "skip_special_tokens": true }, "system_prompt": null, "cache_dir": "~/.cache/huggingface/lighteval", "tokenizer": null, "revision": "main", "dtype": "bfloat16", "tensor_parallel_size": 1, "data_parallel_size": 1, "pipeline_parallel_size": 1, "gpu_memory_utilization": 0.9, "enable_prefix_caching": null, "max_model_length": null, "quantization": null, "load_format": null, "swap_space": 4, "seed": 1234, "trust_remote_code": true, "add_special_tokens": true, "multichoice_continuations_start_space": true, "pairwise_tokenization": false, "max_num_seqs": 128, "max_num_batched_tokens": 2048, "subfolder": null, "is_async": false, "override_chat_template": null, "enforce_eager": false }, "model_name": "models/ministral-base" }, "versions": {}, "results": { "e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": { "acc": 0.25109170305676853, "acc_stderr": 0.020284878098207157, "em_relaxed": 0.25327510917030566, "em_relaxed_stderr": 0.020343162564319364 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { "acc": 0.19638242894056848, "acc_stderr": 0.020220056896337255, "em_relaxed": 0.19896640826873385, "em_relaxed_stderr": 0.02031990120797851 }, "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { "acc": 0.12337662337662338, "acc_stderr": 0.01876954635295657, "em_relaxed": 0.12337662337662338, "em_relaxed_stderr": 0.01876954635295657 }, "scio.prompt1_mqa_single_correct_sk:mat|0": { "acc": 0.23497267759562843, "acc_stderr": 0.022192218701649325, "em_relaxed": 0.2459016393442623, "em_relaxed_stderr": 0.02253970761756048 }, "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { "acc": 0.1981981981981982, "acc_stderr": 0.026815582053277193, "em_relaxed": 0.20270270270270271, "em_relaxed_stderr": 0.027042309486278813 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { "acc": 0.20522388059701493, "acc_stderr": 0.017460611985170203, "em_relaxed": 0.21082089552238806, "em_relaxed_stderr": 0.017634685741009386 }, "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { "acc": 0.16666666666666666, "acc_stderr": 0.02506903489984115, "em_relaxed": 0.1891891891891892, "em_relaxed_stderr": 0.02634582401273633 }, "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { "acc": 0.1891891891891892, "acc_stderr": 0.02438361375395457, "em_relaxed": 0.1891891891891892, "em_relaxed_stderr": 0.02438361375395457 }, "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { "acc": 0.2644927536231884, "acc_stderr": 0.026597083702337265, "em_relaxed": 0.26811594202898553, "em_relaxed_stderr": 0.02671259720934512 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { "acc": 0.2878787878787879, "acc_stderr": 0.03225883512300992, "em_relaxed": 0.2878787878787879, "em_relaxed_stderr": 0.03225883512300992 }, "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { "acc": 0.17, "acc_stderr": 0.03775251680686371, "em_relaxed": 0.17, "em_relaxed_stderr": 0.03775251680686371 }, "slovak_tests.prompt3_qa_sk:maturita|0": { "qem": 0.01327433628318584, "qem_stderr": 0.007629803632081149, "qf1": 0.0022123893805309734, "qf1_stderr": 0.0015609152277667225, "rouge1": 0.0013282561753440866, "rouge1_stderr": 0.0009107295550233445, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0013282561753440866, "rougeL_stderr": 0.0009107295550233445 }, "slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": { "acc": 0.24423963133640553, "acc_stderr": 0.029232990766796058, "em_relaxed": 0.24423963133640553, "em_relaxed_stderr": 0.029232990766796058 }, "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { "acc": 0.25, "acc_stderr": 0.033113308926626096, "em_relaxed": 0.25, "em_relaxed_stderr": 0.033113308926626096 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { "acc": 0.3939393939393939, "acc_stderr": 0.04935824351078519, "em_relaxed": 0.41414141414141414, "em_relaxed_stderr": 0.04975740158605099 }, "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { "acc": 0.16964285714285715, "acc_stderr": 0.035623678500953895, "em_relaxed": 0.1875, "em_relaxed_stderr": 0.0370468111477387 }, "mmlu.prompt2_mqa_single_correct_sk:virology|0": { "acc": 0.17682926829268292, "acc_stderr": 0.02988327785748596, "em_relaxed": 0.18902439024390244, "em_relaxed_stderr": 0.030666839281449514 }, "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { "acc": 0.2462686567164179, "acc_stderr": 0.037358294208002164, "em_relaxed": 0.2537313432835821, "em_relaxed_stderr": 0.037731914327133735 }, "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { "acc": 0.29, "acc_stderr": 0.045604802157206845, "em_relaxed": 0.3, "em_relaxed_stderr": 0.046056618647183814 }, "e_tests.prompt2_mqa_single_correct_sk:geografia|0": { "acc": 0.2857142857142857, "acc_stderr": 0.06520506636966263, "em_relaxed": 0.2857142857142857, "em_relaxed_stderr": 0.06520506636966263 }, "e_tests.prompt2_mqa_single_correct_sk:chemia|0": { "acc": 0.07142857142857142, "acc_stderr": 0.034726602486028414, "em_relaxed": 0.08928571428571429, "em_relaxed_stderr": 0.038450387280282494 }, "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { "acc": 0.1656441717791411, "acc_stderr": 0.020621585415044594, "em_relaxed": 0.17484662576687116, "em_relaxed_stderr": 0.021069502803436863 }, "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { "acc": 0.1875, "acc_stderr": 0.0370468111477387, "em_relaxed": 0.19642857142857142, "em_relaxed_stderr": 0.03770970049347017 }, "movie_review.prompt2_sentiment_binary_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "e_tests.prompt1_mqa_single_correct_sk:biologia|0": { "acc": 0.19540229885057472, "acc_stderr": 0.0427567811097387, "em_relaxed": 0.20689655172413793, "em_relaxed_stderr": 0.04368097459950702 }, "mmlu.prompt3_mqa_single_correct_sk:management|0": { "acc": 0.21, "acc_stderr": 0.040936018074033256, "em_relaxed": 0.21, "em_relaxed_stderr": 0.040936018074033256 }, "e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { "acc": 0.1565217391304348, "acc_stderr": 0.034030776100725475, "em_relaxed": 0.2, "em_relaxed_stderr": 0.03746343246326772 }, "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { "acc": 0.22, "acc_stderr": 0.04163331998932268, "em_relaxed": 0.22, "em_relaxed_stderr": 0.04163331998932268 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { "acc": 0.22727272727272727, "acc_stderr": 0.029857515673386393, "em_relaxed": 0.23232323232323232, "em_relaxed_stderr": 0.030088629490217483 }, "movie_review.prompt3_sentiment_binary_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slovak_tests.prompt2_qa_sk:maturita|0": { "qem": 0.01327433628318584, "qem_stderr": 0.007629803632081149, "qf1": 0.002753195673549656, "qf1_stderr": 0.0020208822149325673, "rouge1": 0.0008509189925119129, "rouge1_stderr": 0.0007009516038640078, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0008509189925119129, "rougeL_stderr": 0.0007009516038640078 }, "e_tests.prompt1_qa_sk:slovensky-jazyk|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0025906735751295338, "qf1_stderr": 0.0025906735751295403, "rouge1": 5.842773836305639e-05, "rouge1_stderr": 4.122343590294019e-05, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 5.842773836305639e-05, "rougeL_stderr": 4.122343590294019e-05 }, "e_tests.prompt2_qa_sk:dejepis|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": { "acc": 0.2205240174672489, "acc_stderr": 0.019394170839568875, "em_relaxed": 0.22270742358078602, "em_relaxed_stderr": 0.019462629121054217 }, "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { "acc": 0.23026315789473684, "acc_stderr": 0.03426059424403165, "em_relaxed": 0.23684210526315788, "em_relaxed_stderr": 0.03459777606810535 }, "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { "acc": 0.2702702702702703, "acc_stderr": 0.02764841916772191, "em_relaxed": 0.28185328185328185, "em_relaxed_stderr": 0.028009688894113486 }, "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { "acc": 0.21100917431192662, "acc_stderr": 0.039262212380368526, "em_relaxed": 0.22018348623853212, "em_relaxed_stderr": 0.03987279758533731 }, "e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": { "acc": 0.045454545454545456, "acc_stderr": 0.031765283723714 }, "slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": { "acc": 0.19815668202764977, "acc_stderr": 0.02712203315362448, "em_relaxed": 0.19815668202764977, "em_relaxed_stderr": 0.02712203315362448 }, "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { "acc": 0.2092426187419769, "acc_stderr": 0.014583336673745648, "em_relaxed": 0.21181001283697048, "em_relaxed_stderr": 0.014648693814173547 }, "e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "e_tests.prompt2_mqa_single_correct_sk:biologia|0": { "acc": 0.14942528735632185, "acc_stderr": 0.03844315810763212, "em_relaxed": 0.14942528735632185, "em_relaxed_stderr": 0.03844315810763212 }, "mmlu.prompt1_mqa_single_correct_sk:virology|0": { "acc": 0.23780487804878048, "acc_stderr": 0.03334645408665337, "em_relaxed": 0.24390243902439024, "em_relaxed_stderr": 0.03363591048272823 }, "e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { "acc": 0.1232876712328767, "acc_stderr": 0.022266919281488138, "em_relaxed": 0.1232876712328767, "em_relaxed_stderr": 0.022266919281488138 }, "e_tests.prompt1_qa_sk:geografia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "sklep.prompt3_nli_3_classes_sk:nli|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "scio.prompt3_mqa_single_correct_sk:vsp|0": { "acc": 0.20388349514563106, "acc_stderr": 0.014043631695334572, "em_relaxed": 0.21723300970873785, "em_relaxed_stderr": 0.014374053054620224 }, "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { "acc": 0.2848101265822785, "acc_stderr": 0.03601957344556447, "em_relaxed": 0.2848101265822785, "em_relaxed_stderr": 0.03601957344556447 }, "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "arc.prompt1_mqa_single_correct_sk|0": { "acc": 0.2728828058169376, "acc_stderr": 0.013033734552785697, "em_relaxed": 0.2728828058169376, "em_relaxed_stderr": 0.013033734552785697 }, "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { "acc": 0.18181818181818182, "acc_stderr": 0.038961038961039, "em_relaxed": 0.18181818181818182, "em_relaxed_stderr": 0.038961038961039 }, "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { "acc": 0.2616822429906542, "acc_stderr": 0.0426929191572811, "em_relaxed": 0.27102803738317754, "em_relaxed_stderr": 0.04317273776566666 }, "belebele.prompt2_mqa_single_correct_sk|0": { "acc": 0.2511111111111111, "acc_stderr": 0.014463114105170805, "em_relaxed": 0.25666666666666665, "em_relaxed_stderr": 0.01456789134238002 }, "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { "acc": 0.20922288642186165, "acc_stderr": 0.0118915566619952, "em_relaxed": 0.21178479931682323, "em_relaxed_stderr": 0.011944744511660903 }, "senti_sk.prompt2_sentiment_3_class_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { "acc": 0.22018348623853212, "acc_stderr": 0.03987279758533728, "em_relaxed": 0.22018348623853212, "em_relaxed_stderr": 0.03987279758533728 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { "acc": 0.13333333333333333, "acc_stderr": 0.029365879728106854, "em_relaxed": 0.13333333333333333, "em_relaxed_stderr": 0.029365879728106854 }, "e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": { "acc": 0.014285714285714285, "acc_stderr": 0.010065123433632259 }, "sklep.prompt2_hate_speech_sk:hate-speech|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slovakcopa.prompt3_copa_sk|0": { "acc": 0.144, "acc_stderr": 0.015716934945725767, "em_relaxed": 0.144, "em_relaxed_stderr": 0.015716934945725767 }, "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { "acc": 0.2852664576802508, "acc_stderr": 0.025321176274679096, "em_relaxed": 0.2852664576802508, "em_relaxed_stderr": 0.025321176274679096 }, "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { "acc": 0.1651376146788991, "acc_stderr": 0.0357288117600854, "em_relaxed": 0.1651376146788991, "em_relaxed_stderr": 0.0357288117600854 }, "e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { "acc": 0.3656716417910448, "acc_stderr": 0.041761576413228745, "em_relaxed": 0.373134328358209, "em_relaxed_stderr": 0.041936678130523646 }, "slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": { "acc": 0.21683673469387754, "acc_stderr": 0.02084031144263498, "em_relaxed": 0.2193877551020408, "em_relaxed_stderr": 0.020928374259338522 }, "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { "acc": 0.24, "acc_stderr": 0.021380899352993928, "em_relaxed": 0.2425, "em_relaxed_stderr": 0.021456591955524264 }, "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { "acc": 0.30638297872340425, "acc_stderr": 0.03013590647851756, "em_relaxed": 0.3148936170212766, "em_relaxed_stderr": 0.03036358219723817 }, "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { "acc": 0.22402597402597402, "acc_stderr": 0.023795979800861743, "em_relaxed": 0.22402597402597402, "em_relaxed_stderr": 0.023795979800861743 }, "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { "acc": 0.2911392405063291, "acc_stderr": 0.03625609398754825, "em_relaxed": 0.2974683544303797, "em_relaxed_stderr": 0.03648408889243667 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { "acc": 0.19689119170984457, "acc_stderr": 0.028697873971860688, "em_relaxed": 0.19689119170984457, "em_relaxed_stderr": 0.028697873971860688 }, "scio.prompt3_mqa_single_correct_sk:mat|0": { "acc": 0.20765027322404372, "acc_stderr": 0.021231382642567846, "em_relaxed": 0.21584699453551912, "em_relaxed_stderr": 0.02153411244027524 }, "e_tests.prompt2_qa_sk:geografia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { "acc": 0.21100917431192662, "acc_stderr": 0.03926221238036852, "em_relaxed": 0.21100917431192662, "em_relaxed_stderr": 0.03926221238036852 }, "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { "acc": 0.17094017094017094, "acc_stderr": 0.02466249684520983, "em_relaxed": 0.17094017094017094, "em_relaxed_stderr": 0.02466249684520983 }, "scio.prompt1_mqa_single_correct_sk:bio|0": { "acc": 0.2743362831858407, "acc_stderr": 0.024268949584718702, "em_relaxed": 0.28023598820059, "em_relaxed_stderr": 0.024428605014452955 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { "acc": 0.24352331606217617, "acc_stderr": 0.03097543638684542, "em_relaxed": 0.24870466321243523, "em_relaxed_stderr": 0.0311958408777003 }, "sklep.prompt3_qa_sk:question-answering|0": { "qem": 0.0002087029114056141, "qem_stderr": 0.0001475675430580104, "qf1": 0.006093522810123945, "qf1_stderr": 0.00032767921496533014 }, "e_tests.prompt3_qa_sk:biologia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0011204481792717084, "rouge1_stderr": 0.0011204481792717086, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0011204481792717084, "rougeL_stderr": 0.0011204481792717086 }, "slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": { "acc": 0.13333333333333333, "acc_stderr": 0.03333333333333327, "em_relaxed": 0.13333333333333333, "em_relaxed_stderr": 0.03333333333333327 }, "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { "acc": 0.23134328358208955, "acc_stderr": 0.03656527469759901, "em_relaxed": 0.23134328358208955, "em_relaxed_stderr": 0.03656527469759901 }, "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { "acc": 0.2182741116751269, "acc_stderr": 0.029505319811543015, "em_relaxed": 0.2182741116751269, "em_relaxed_stderr": 0.029505319811543015 }, "e_tests.prompt2_mqa_single_correct_sk:matematika|0": { "acc": 0.10256410256410256, "acc_stderr": 0.04921614717627658, "em_relaxed": 0.3076923076923077, "em_relaxed_stderr": 0.074871425137275 }, "scio.prompt1_mqa_single_correct_sk:aj|0": { "acc": 0.6, "acc_stderr": 0.16329931618554522, "em_relaxed": 0.6, "em_relaxed_stderr": 0.16329931618554522 }, "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { "acc": 0.22393822393822393, "acc_stderr": 0.02595388693967799, "em_relaxed": 0.23166023166023167, "em_relaxed_stderr": 0.026265915716011466 }, "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { "acc": 0.2, "acc_stderr": 0.04020151261036846, "em_relaxed": 0.2, "em_relaxed_stderr": 0.04020151261036846 }, "financial_exams.prompt1_completion_sk|0": { "acc": 0.053973013493253376, "acc_stderr": 0.006189064679835392, "em_relaxed": 0.05697151424287856, "em_relaxed_stderr": 0.006348574383669881 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { "acc": 0.28110599078341014, "acc_stderr": 0.03058726229145915, "em_relaxed": 0.2903225806451613, "em_relaxed_stderr": 0.030884745411494743 }, "slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0012698412698412698, "rouge1_stderr": 0.00126984126984127, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0012698412698412698, "rougeL_stderr": 0.00126984126984127 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { "acc": 0.22807017543859648, "acc_stderr": 0.056069817847611766, "em_relaxed": 0.22807017543859648, "em_relaxed_stderr": 0.056069817847611766 }, "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { "acc": 0.2753623188405797, "acc_stderr": 0.02693682252998795, "em_relaxed": 0.27898550724637683, "em_relaxed_stderr": 0.027045590734089176 }, "ifeval_sk.ifeval|0": { "inst_level_loose_acc": 0.17312348668280872, "inst_level_loose_acc_stderr": 0.0004210039776560394, "inst_level_strict_acc": 0.17312348668280872, "inst_level_strict_acc_stderr": 0.0004210039776560394, "prompt_level_loose_acc": 0.0782122905027933, "prompt_level_loose_acc_stderr": 0.011597665111537078, "prompt_level_strict_acc": 0.0782122905027933, "prompt_level_strict_acc_stderr": 0.011597665111537078 }, "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { "acc": 0.2826086956521739, "acc_stderr": 0.027152165210427823, "em_relaxed": 0.286231884057971, "em_relaxed_stderr": 0.02725657169175206 }, "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { "acc": 0.2314261315115286, "acc_stderr": 0.012329801197790603, "em_relaxed": 0.23825789923142612, "em_relaxed_stderr": 0.012454741062931133 }, "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { "acc": 0.1834862385321101, "acc_stderr": 0.03724531275523975, "em_relaxed": 0.1834862385321101, "em_relaxed_stderr": 0.03724531275523975 }, "movie_review.prompt1_sentiment_binary_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slovak_tests.prompt1_qa_sk:monitor9|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { "acc": 0.2416555407209613, "acc_stderr": 0.01565239642514941, "em_relaxed": 0.2416555407209613, "em_relaxed_stderr": 0.01565239642514941 }, "slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": { "acc": 0.15718157181571815, "acc_stderr": 0.018973345526861658, "em_relaxed": 0.16260162601626016, "em_relaxed_stderr": 0.019235549151947248 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { "acc": 0.22897196261682243, "acc_stderr": 0.028789653442089263, "em_relaxed": 0.24766355140186916, "em_relaxed_stderr": 0.02957653529316449 }, "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { "acc": 0.17391304347826086, "acc_stderr": 0.03549985134468377, "em_relaxed": 0.20869565217391303, "em_relaxed_stderr": 0.03806063686277675 }, "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { "acc": 0.21983914209115282, "acc_stderr": 0.02147203067557281, "em_relaxed": 0.21983914209115282, "em_relaxed_stderr": 0.02147203067557281 }, "toxic_sk.prompt1_toxicity_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "driving_test.prompt2_mqa_single_correct_sk|0": { "acc": 0.09985734664764621, "acc_stderr": 0.011331741573886145, "em_relaxed": 0.10271041369472182, "em_relaxed_stderr": 0.011474255943096628 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { "acc": 0.32323232323232326, "acc_stderr": 0.04724590344515123, "em_relaxed": 0.3333333333333333, "em_relaxed_stderr": 0.04761904761904759 }, "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { "acc": 0.19736842105263158, "acc_stderr": 0.03238981601699397, "em_relaxed": 0.21052631578947367, "em_relaxed_stderr": 0.033176727875331574 }, "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { "acc": 0.2803030303030303, "acc_stderr": 0.027695605168511307, "em_relaxed": 0.2878787878787879, "em_relaxed_stderr": 0.027919261062611087 }, "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { "acc": 0.2322801024765158, "acc_stderr": 0.012345664537818761, "em_relaxed": 0.23654995730145176, "em_relaxed_stderr": 0.012423924913578318 }, "sklep.prompt2_nli_3_classes_sk:nli|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { "acc": 0.2119815668202765, "acc_stderr": 0.02780931949344348, "em_relaxed": 0.22580645161290322, "em_relaxed_stderr": 0.028448938828651484 }, "sklep.prompt1_nli_binary_sk:rte|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "e_tests.prompt3_qa_sk:dejepis|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { "acc": 0.22, "acc_stderr": 0.0416333199893227, "em_relaxed": 0.24, "em_relaxed_stderr": 0.04292346959909283 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { "acc": 0.2631578947368421, "acc_stderr": 0.05884389414473131, "em_relaxed": 0.2807017543859649, "em_relaxed_stderr": 0.06004585739704729 }, "slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": { "acc": 0.13520408163265307, "acc_stderr": 0.017292721140889967, "em_relaxed": 0.1377551020408163, "em_relaxed_stderr": 0.01742933367587574 }, "scio.prompt3_mqa_single_correct_sk:che|0": { "acc": 0.22764227642276422, "acc_stderr": 0.026788763286110986, "em_relaxed": 0.23577235772357724, "em_relaxed_stderr": 0.027119068752977968 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { "acc": 0.23333333333333334, "acc_stderr": 0.03464962582372412, "em_relaxed": 0.23333333333333334, "em_relaxed_stderr": 0.03464962582372412 }, "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { "acc": 0.1919191919191919, "acc_stderr": 0.03978080447933685, "em_relaxed": 0.1919191919191919, "em_relaxed_stderr": 0.03978080447933685 }, "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { "acc": 0.16071428571428573, "acc_stderr": 0.034859460964757394, "em_relaxed": 0.16071428571428573, "em_relaxed_stderr": 0.034859460964757394 }, "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { "acc": 0.2535211267605634, "acc_stderr": 0.03663585085277138, "em_relaxed": 0.2605633802816901, "em_relaxed_stderr": 0.03696558732615873 }, "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { "acc": 0.22442244224422442, "acc_stderr": 0.024007242951044432, "em_relaxed": 0.25412541254125415, "em_relaxed_stderr": 0.02505263910183634 }, "slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.001330671989354624, "rouge1_stderr": 0.0013306719893546237, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.001330671989354624, "rougeL_stderr": 0.0013306719893546237 }, "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { "acc": 0.18711656441717792, "acc_stderr": 0.021633588964875924, "em_relaxed": 0.19325153374233128, "em_relaxed_stderr": 0.02190225688747218 }, "e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": { "acc": 0.21615720524017468, "acc_stderr": 0.0192548989397317, "em_relaxed": 0.2183406113537118, "em_relaxed_stderr": 0.01932493025894029 }, "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { "acc": 0.14, "acc_stderr": 0.0348735088019777, "em_relaxed": 0.16, "em_relaxed_stderr": 0.03684529491774708 }, "e_tests.prompt1_mqa_single_correct_sk:geografia|0": { "acc": 0.20408163265306123, "acc_stderr": 0.058172215566282555, "em_relaxed": 0.20408163265306123, "em_relaxed_stderr": 0.058172215566282555 }, "e_tests.prompt2_qa_sk:fyzika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "multi_wiki_qa.prompt3_qa_sk|0": { "qem": 0.0006003602161296778, "qem_stderr": 0.00034654874631857635, "qf1": 0.005765729879562343, "qf1_stderr": 0.0005521722468476828 }, "e_tests.prompt1_qa_sk:fyzika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { "acc": 0.19172932330827067, "acc_stderr": 0.024182419147367247, "em_relaxed": 0.20676691729323307, "em_relaxed_stderr": 0.024878145232561027 }, "e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": { "acc": 0.15942028985507245, "acc_stderr": 0.04439221265797129, "em_relaxed": 0.15942028985507245, "em_relaxed_stderr": 0.04439221265797129 }, "e_tests.prompt2_mqa_single_correct_sk:dejepis|0": { "acc": 0.20833333333333334, "acc_stderr": 0.059238169759886904, "em_relaxed": 0.20833333333333334, "em_relaxed_stderr": 0.059238169759886904 }, "scio.prompt1_mqa_single_correct_sk:vsp|0": { "acc": 0.24150485436893204, "acc_stderr": 0.01491898981873877, "em_relaxed": 0.25849514563106796, "em_relaxed_stderr": 0.015261010021381633 }, "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { "acc": 0.16, "acc_stderr": 0.03684529491774709, "em_relaxed": 0.17, "em_relaxed_stderr": 0.0377525168068637 }, "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { "acc": 0.3050847457627119, "acc_stderr": 0.04256799926288002, "em_relaxed": 0.3050847457627119, "em_relaxed_stderr": 0.04256799926288002 }, "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { "acc": 0.288135593220339, "acc_stderr": 0.04187011593049808, "em_relaxed": 0.2966101694915254, "em_relaxed_stderr": 0.04222776832233627 }, "slovakcopa.prompt2_copa_sk|0": { "acc": 0.006, "acc_stderr": 0.003457152557758384, "em_relaxed": 0.006, "em_relaxed_stderr": 0.003457152557758384 }, "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { "acc": 0.2116564417177914, "acc_stderr": 0.022658534978587233, "em_relaxed": 0.22085889570552147, "em_relaxed_stderr": 0.023010381863796215 }, "e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { "acc": 0.24, "acc_stderr": 0.042923469599092816, "em_relaxed": 0.25, "em_relaxed_stderr": 0.04351941398892446 }, "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { "acc": 0.2807017543859649, "acc_stderr": 0.034462962170884265, "em_relaxed": 0.2807017543859649, "em_relaxed_stderr": 0.034462962170884265 }, "e_tests.prompt1_qa_sk:biologia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { "acc": 0.15, "acc_stderr": 0.03588702812826371, "em_relaxed": 0.15, "em_relaxed_stderr": 0.03588702812826371 }, "e_tests.prompt3_mqa_single_correct_sk:chemia|0": { "acc": 0.07142857142857142, "acc_stderr": 0.03472660248602842, "em_relaxed": 0.08928571428571429, "em_relaxed_stderr": 0.038450387280282494 }, "e_tests.prompt1_mqa_single_correct_sk:matematika|0": { "acc": 0.15384615384615385, "acc_stderr": 0.05852968540405249, "em_relaxed": 0.358974358974359, "em_relaxed_stderr": 0.07781756136754926 }, "e_tests.prompt3_qa_sk:geografia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": { "acc": 0.045454545454545456, "acc_stderr": 0.031765283723714 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { "acc": 0.2383177570093458, "acc_stderr": 0.029192770642101563, "em_relaxed": 0.2570093457943925, "em_relaxed_stderr": 0.029941691533244646 }, "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { "acc": 0.24932975871313673, "acc_stderr": 0.022430563714050267, "em_relaxed": 0.24932975871313673, "em_relaxed_stderr": 0.022430563714050267 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { "acc": 0.24468085106382978, "acc_stderr": 0.025645553622266722, "em_relaxed": 0.25177304964539005, "em_relaxed_stderr": 0.025892151156709405 }, "e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { "acc": 0.22535211267605634, "acc_stderr": 0.035186286213623755, "em_relaxed": 0.22535211267605634, "em_relaxed_stderr": 0.035186286213623755 }, "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { "acc": 0.296551724137931, "acc_stderr": 0.03806142687309993, "em_relaxed": 0.3103448275862069, "em_relaxed_stderr": 0.03855289616378948 }, "e_tests.prompt1_mqa_single_correct_sk:chemia|0": { "acc": 0.10714285714285714, "acc_stderr": 0.0417053005800816, "em_relaxed": 0.10714285714285714, "em_relaxed_stderr": 0.0417053005800816 }, "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { "acc": 0.25671140939597314, "acc_stderr": 0.01790782476183757, "em_relaxed": 0.25838926174496646, "em_relaxed_stderr": 0.017945962358551722 }, "e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": { "acc": 0.045454545454545456, "acc_stderr": 0.031765283723714 }, "e_tests.prompt3_qa_sk:obcianska-nauka|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { "acc": 0.2, "acc_stderr": 0.04020151261036843, "em_relaxed": 0.22, "em_relaxed_stderr": 0.041633319989322716 }, "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { "acc": 0.2018348623853211, "acc_stderr": 0.03862180715692686, "em_relaxed": 0.2018348623853211, "em_relaxed_stderr": 0.03862180715692686 }, "e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": { "acc": 0.14492753623188406, "acc_stderr": 0.0426896366873121, "em_relaxed": 0.14492753623188406, "em_relaxed_stderr": 0.0426896366873121 }, "e_tests.prompt3_mqa_single_correct_sk:fyzika|0": { "acc": 0.18181818181818182, "acc_stderr": 0.08416546361568647, "em_relaxed": 0.2727272727272727, "em_relaxed_stderr": 0.0971859061499725 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { "acc": 0.16917293233082706, "acc_stderr": 0.023030205866522328, "em_relaxed": 0.17669172932330826, "em_relaxed_stderr": 0.023429683004809066 }, "e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": { "acc": 0.0, "acc_stderr": 0.0 }, "e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": { "acc": 0.034482758620689655, "acc_stderr": 0.034482758620689676 }, "e_tests.prompt3_qa_sk:chemia|0": { "qem": 0.018518518518518517, "qem_stderr": 0.018518518518518517, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0006660882976672449, "rouge1_stderr": 0.0004499203448417917, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0006660882976672449, "rougeL_stderr": 0.0004499203448417917 }, "e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": { "acc": 0.03571428571428571, "acc_stderr": 0.02502318034804895 }, "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { "acc": 0.1826086956521739, "acc_stderr": 0.03618456100987082, "em_relaxed": 0.21739130434782608, "em_relaxed_stderr": 0.038631448549506 }, "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { "acc": 0.21379310344827587, "acc_stderr": 0.034165204477475494, "em_relaxed": 0.2206896551724138, "em_relaxed_stderr": 0.03455930201924811 }, "e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": { "acc": 0.014285714285714285, "acc_stderr": 0.010065123433632259 }, "e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { "acc": 0.22, "acc_stderr": 0.041633319989322695, "em_relaxed": 0.24, "em_relaxed_stderr": 0.042923469599092816 }, "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { "acc": 0.24786324786324787, "acc_stderr": 0.028286324075564397, "em_relaxed": 0.25213675213675213, "em_relaxed_stderr": 0.02844796547623102 }, "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { "acc": 0.26900584795321636, "acc_stderr": 0.0340105262010409, "em_relaxed": 0.27485380116959063, "em_relaxed_stderr": 0.034240429246915824 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { "acc": 0.21717171717171718, "acc_stderr": 0.02937661648494564, "em_relaxed": 0.21717171717171718, "em_relaxed_stderr": 0.02937661648494564 }, "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { "acc": 0.26702269692923897, "acc_stderr": 0.01617590372442817, "em_relaxed": 0.26702269692923897, "em_relaxed_stderr": 0.01617590372442817 }, "e_tests.prompt2_qa_sk:informatika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.010073260073260074, "qf1_stderr": 0.007341600122632042, "rouge1": 0.0012475950143262986, "rouge1_stderr": 0.000686304468853021, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0012475950143262986, "rougeL_stderr": 0.000686304468853021 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { "acc": 0.2474747474747475, "acc_stderr": 0.030746300742124498, "em_relaxed": 0.25252525252525254, "em_relaxed_stderr": 0.030954055470365897 }, "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { "acc": 0.24242424242424243, "acc_stderr": 0.04329004329004326, "em_relaxed": 0.26262626262626265, "em_relaxed_stderr": 0.04445287676983944 }, "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { "acc": 0.16964285714285715, "acc_stderr": 0.0356236785009539, "em_relaxed": 0.19642857142857142, "em_relaxed_stderr": 0.03770970049347018 }, "mmlu.prompt3_mqa_single_correct_sk:virology|0": { "acc": 0.22560975609756098, "acc_stderr": 0.03273897454566342, "em_relaxed": 0.22560975609756098, "em_relaxed_stderr": 0.03273897454566342 }, "slovakcopa.prompt1_copa_sk|0": { "acc": 0.01, "acc_stderr": 0.0044541754970721, "em_relaxed": 0.01, "em_relaxed_stderr": 0.0044541754970721 }, "scio.prompt3_mqa_single_correct_sk:bio|0": { "acc": 0.3008849557522124, "acc_stderr": 0.024946878423156852, "em_relaxed": 0.30973451327433627, "em_relaxed_stderr": 0.025150378763664 }, "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { "acc": 0.205607476635514, "acc_stderr": 0.03925401580070484, "em_relaxed": 0.21495327102803738, "em_relaxed_stderr": 0.03989944463395407 }, "multi_wiki_qa.prompt2_qa_sk|0": { "qem": 0.0006003602161296778, "qem_stderr": 0.0003465487463185779, "qf1": 0.005417769336679904, "qf1_stderr": 0.0005509525930025259 }, "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { "acc": 0.21568627450980393, "acc_stderr": 0.04092563958237654, "em_relaxed": 0.22549019607843138, "em_relaxed_stderr": 0.04158307533083286 }, "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { "acc": 0.20425531914893616, "acc_stderr": 0.026355158413349424, "em_relaxed": 0.20425531914893616, "em_relaxed_stderr": 0.026355158413349424 }, "e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": { "acc": 0.18840579710144928, "acc_stderr": 0.047420064740574204, "em_relaxed": 0.18840579710144928, "em_relaxed_stderr": 0.047420064740574204 }, "e_tests.prompt3_mqa_single_correct_sk:dejepis|0": { "acc": 0.20833333333333334, "acc_stderr": 0.059238169759886904, "em_relaxed": 0.20833333333333334, "em_relaxed_stderr": 0.059238169759886904 }, "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { "acc": 0.19122257053291536, "acc_stderr": 0.022053144718757862, "em_relaxed": 0.19122257053291536, "em_relaxed_stderr": 0.022053144718757862 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { "acc": 0.23333333333333334, "acc_stderr": 0.03464962582372412, "em_relaxed": 0.23333333333333334, "em_relaxed_stderr": 0.03464962582372412 }, "sklep.prompt1_hate_speech_sk:hate-speech|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "driving_test.prompt1_mqa_single_correct_sk|0": { "acc": 0.16405135520684735, "acc_stderr": 0.01399686230459443, "em_relaxed": 0.16833095577746077, "em_relaxed_stderr": 0.014141915666582279 }, "e_tests.prompt3_mqa_single_correct_sk:informatika|0": { "acc": 0.256, "acc_stderr": 0.03919183588453089, "em_relaxed": 0.264, "em_relaxed_stderr": 0.03958494337416042 }, "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { "acc": 0.18959731543624161, "acc_stderr": 0.01606971018943585, "em_relaxed": 0.19463087248322147, "em_relaxed_stderr": 0.0162309847870997 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { "acc": 0.22429906542056074, "acc_stderr": 0.028580583273338626, "em_relaxed": 0.2383177570093458, "em_relaxed_stderr": 0.029192770642101563 }, "e_tests.prompt1_mqa_single_correct_sk:fyzika|0": { "acc": 0.13636363636363635, "acc_stderr": 0.07488677009526491, "em_relaxed": 0.22727272727272727, "em_relaxed_stderr": 0.09144861547306321 }, "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { "acc": 0.26744186046511625, "acc_stderr": 0.03384836428157859, "em_relaxed": 0.27906976744186046, "em_relaxed_stderr": 0.03430085607014882 }, "slovak_tests.prompt3_qa_sk:monitor9|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { "acc": 0.136986301369863, "acc_stderr": 0.023287301596102452, "em_relaxed": 0.136986301369863, "em_relaxed_stderr": 0.023287301596102452 }, "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { "acc": 0.2025, "acc_stderr": 0.0201183061066658, "em_relaxed": 0.205, "em_relaxed_stderr": 0.02021035988339997 }, "sklep.prompt2_qa_sk:question-answering|0": { "qem": 0.0002087029114056141, "qem_stderr": 0.0001475675430580104, "qf1": 0.006743145308527306, "qf1_stderr": 0.00039734127192685853 }, "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { "acc": 0.04884318766066838, "acc_stderr": 0.00773245768018304, "em_relaxed": 0.08354755784061697, "em_relaxed_stderr": 0.009926847196525902 }, "senti_sk.prompt3_sentiment_binary_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "scio.prompt2_mqa_single_correct_sk:vsp|0": { "acc": 0.23786407766990292, "acc_stderr": 0.014841600192116697, "em_relaxed": 0.2512135922330097, "em_relaxed_stderr": 0.015118218765838777 }, "e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "e_tests.prompt3_qa_sk:slovensky-jazyk|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0025906735751295338, "qf1_stderr": 0.0025906735751295403, "rouge1": 0.00035708568846413394, "rouge1_stderr": 0.00020841637674444064, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.00035708568846413394, "rougeL_stderr": 0.00020841637674444064 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { "acc": 0.24561403508771928, "acc_stderr": 0.05752139056405265, "em_relaxed": 0.2631578947368421, "em_relaxed_stderr": 0.05884389414473131 }, "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { "acc": 0.16595744680851063, "acc_stderr": 0.02432117475103868, "em_relaxed": 0.16595744680851063, "em_relaxed_stderr": 0.02432117475103868 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { "acc": 0.2126865671641791, "acc_stderr": 0.01769159425053082, "em_relaxed": 0.21641791044776118, "em_relaxed_stderr": 0.01780376934180082 }, "senti_sk.prompt2_sentiment_binary_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "scio.prompt1_mqa_single_correct_sk:che|0": { "acc": 0.25609756097560976, "acc_stderr": 0.027885450835842325, "em_relaxed": 0.25609756097560976, "em_relaxed_stderr": 0.027885450835842325 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { "acc": 0.24444444444444444, "acc_stderr": 0.03212157057535212, "em_relaxed": 0.25, "em_relaxed_stderr": 0.032364888900157734 }, "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { "acc": 0.14285714285714285, "acc_stderr": 0.03321361106966269, "em_relaxed": 0.16071428571428573, "em_relaxed_stderr": 0.0348594609647574 }, "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { "acc": 0.21212121212121213, "acc_stderr": 0.04129606932540889, "em_relaxed": 0.24242424242424243, "em_relaxed_stderr": 0.04329004329004327 }, "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { "acc": 0.26959247648902823, "acc_stderr": 0.024884157592464488, "em_relaxed": 0.26959247648902823, "em_relaxed_stderr": 0.024884157592464488 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { "acc": 0.2538860103626943, "acc_stderr": 0.03141024780565318, "em_relaxed": 0.2538860103626943, "em_relaxed_stderr": 0.03141024780565318 }, "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { "acc": 0.21212121212121213, "acc_stderr": 0.04129606932540889, "em_relaxed": 0.23232323232323232, "em_relaxed_stderr": 0.042660160170546885 }, "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { "acc": 0.042416452442159386, "acc_stderr": 0.0072301115841424965, "em_relaxed": 0.07840616966580977, "em_relaxed_stderr": 0.009643493430258573 }, "belebele.prompt3_mqa_single_correct_sk|0": { "acc": 0.24666666666666667, "acc_stderr": 0.014377023375409347, "em_relaxed": 0.2544444444444444, "em_relaxed_stderr": 0.014526354751055178 }, "arc.prompt3_mqa_single_correct_sk|0": { "acc": 0.22840034217279725, "acc_stderr": 0.012283523666042804, "em_relaxed": 0.22840034217279725, "em_relaxed_stderr": 0.012283523666042804 }, "financial_exams.prompt2_completion_sk|0": { "acc": 0.07871064467766117, "acc_stderr": 0.00737563958812383, "em_relaxed": 0.07946026986506746, "em_relaxed_stderr": 0.007407662917614668 }, "e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { "acc": 0.23049645390070922, "acc_stderr": 0.025123739226872395, "em_relaxed": 0.24113475177304963, "em_relaxed_stderr": 0.02551873104953778 }, "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { "acc": 0.24, "acc_stderr": 0.042923469599092816, "em_relaxed": 0.24, "em_relaxed_stderr": 0.042923469599092816 }, "e_tests.prompt3_mqa_single_correct_sk:biologia|0": { "acc": 0.10344827586206896, "acc_stderr": 0.03283976780985287, "em_relaxed": 0.11494252873563218, "em_relaxed_stderr": 0.03439354046699091 }, "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { "acc": 0.24299065420560748, "acc_stderr": 0.041657429989652724, "em_relaxed": 0.24299065420560748, "em_relaxed_stderr": 0.041657429989652724 }, "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { "acc": 0.2191780821917808, "acc_stderr": 0.028018592326462144, "em_relaxed": 0.2191780821917808, "em_relaxed_stderr": 0.028018592326462144 }, "slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": { "acc": 0.18157181571815717, "acc_stderr": 0.020095119692986566, "em_relaxed": 0.18157181571815717, "em_relaxed_stderr": 0.020095119692986566 }, "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { "acc": 0.22549019607843138, "acc_stderr": 0.04158307533083286, "em_relaxed": 0.23529411764705882, "em_relaxed_stderr": 0.04220773659171455 }, "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { "acc": 0.20535714285714285, "acc_stderr": 0.03834241021419071, "em_relaxed": 0.22321428571428573, "em_relaxed_stderr": 0.039523019677025116 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { "acc": 0.1921641791044776, "acc_stderr": 0.01703416618213854, "em_relaxed": 0.1958955223880597, "em_relaxed_stderr": 0.01715898536836359 }, "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { "acc": 0.23529411764705882, "acc_stderr": 0.04220773659171455, "em_relaxed": 0.24509803921568626, "em_relaxed_stderr": 0.042801058373643945 }, "e_tests.prompt1_mqa_single_correct_sk:dejepis|0": { "acc": 0.22916666666666666, "acc_stderr": 0.06130657275434794, "em_relaxed": 0.22916666666666666, "em_relaxed_stderr": 0.06130657275434794 }, "scio.prompt2_mqa_single_correct_sk:bio|0": { "acc": 0.2861356932153392, "acc_stderr": 0.02458303513041921, "em_relaxed": 0.2949852507374631, "em_relaxed_stderr": 0.024805095426679618 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { "acc": 0.23255813953488372, "acc_stderr": 0.021502792400356754, "em_relaxed": 0.24031007751937986, "em_relaxed_stderr": 0.02174755921996615 }, "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { "acc": 0.2537878787878788, "acc_stderr": 0.02683420376859365, "em_relaxed": 0.25757575757575757, "em_relaxed_stderr": 0.026965016533893105 }, "toxic_sk.prompt3_toxicity_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { "acc": 0.17777777777777778, "acc_stderr": 0.033027898599017176, "em_relaxed": 0.17777777777777778, "em_relaxed_stderr": 0.033027898599017176 }, "driving_test.prompt3_mqa_single_correct_sk|0": { "acc": 0.14835948644793154, "acc_stderr": 0.013434972977209091, "em_relaxed": 0.15121255349500715, "em_relaxed_stderr": 0.01354080168340582 }, "slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": { "acc": 0.1760204081632653, "acc_stderr": 0.019259781722629446, "em_relaxed": 0.1760204081632653, "em_relaxed_stderr": 0.019259781722629446 }, "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { "acc": 0.19, "acc_stderr": 0.019639610121239284, "em_relaxed": 0.1925, "em_relaxed_stderr": 0.01973786547054499 }, "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { "acc": 0.12337662337662338, "acc_stderr": 0.018769546352956558, "em_relaxed": 0.12337662337662338, "em_relaxed_stderr": 0.018769546352956558 }, "sklep.prompt1_qa_sk:question-answering|0": { "qem": 0.0004174058228112282, "qem_stderr": 0.00020867023775979663, "qf1": 0.008276210006991061, "qf1_stderr": 0.00042241089189288785 }, "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { "acc": 0.2, "acc_stderr": 0.04020151261036846, "em_relaxed": 0.22, "em_relaxed_stderr": 0.041633319989322674 }, "skj1.prompt1_skj1_sk|0": { "acc": 0.41, "acc_stderr": 0.04943110704237102, "em_relaxed": 0.44, "em_relaxed_stderr": 0.049888765156985884 }, "e_tests.prompt2_qa_sk:slovensky-jazyk|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0025906735751295338, "qf1_stderr": 0.0025906735751295403, "rouge1": 0.00011405864075384238, "rouge1_stderr": 8.908193183205394e-05, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.00011405864075384238, "rougeL_stderr": 8.908193183205394e-05 }, "belebele.prompt1_mqa_single_correct_sk|0": { "acc": 0.2633333333333333, "acc_stderr": 0.014689553047342515, "em_relaxed": 0.2722222222222222, "em_relaxed_stderr": 0.014845038794433947 }, "e_tests.prompt2_qa_sk:chemia|0": { "qem": 0.018518518518518517, "qem_stderr": 0.018518518518518517, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 3.292181069958847e-05, "rouge1_stderr": 3.292181069958849e-05, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 3.292181069958847e-05, "rougeL_stderr": 3.292181069958849e-05 }, "sklep.prompt2_nli_binary_sk:rte|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "multi_wiki_qa.prompt1_qa_sk|0": { "qem": 0.00040024014408645187, "qem_stderr": 0.00028298419465801113, "qf1": 0.004608242571473309, "qf1_stderr": 0.00046536745363106924 }, "slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": { "acc": 0.21904761904761905, "acc_stderr": 0.040556911537178254, "em_relaxed": 0.21904761904761905, "em_relaxed_stderr": 0.040556911537178254 }, "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { "acc": 0.17, "acc_stderr": 0.03775251680686371, "em_relaxed": 0.17, "em_relaxed_stderr": 0.03775251680686371 }, "skj1.prompt3_skj1_sk|0": { "acc": 0.75, "acc_stderr": 0.04351941398892446, "em_relaxed": 0.75, "em_relaxed_stderr": 0.04351941398892446 }, "slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0021574973031283713, "rouge1_stderr": 0.0021574973031283713, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0010787486515641857, "rougeL_stderr": 0.0010787486515641857 }, "e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": { "acc": 0.034482758620689655, "acc_stderr": 0.034482758620689676 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { "acc": 0.22777777777777777, "acc_stderr": 0.03134731210385172, "em_relaxed": 0.23333333333333334, "em_relaxed_stderr": 0.031612960436120094 }, "scio.prompt2_mqa_single_correct_sk:mat|0": { "acc": 0.21311475409836064, "acc_stderr": 0.021434631795998253, "em_relaxed": 0.22131147540983606, "em_relaxed_stderr": 0.021728884019137824 }, "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { "acc": 0.20930232558139536, "acc_stderr": 0.03110958390976464, "em_relaxed": 0.21511627906976744, "em_relaxed_stderr": 0.03142253684735939 }, "sklep.prompt3_hate_speech_sk:hate-speech|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { "acc": 0.18666666666666668, "acc_stderr": 0.03192085664169663, "em_relaxed": 0.19333333333333333, "em_relaxed_stderr": 0.03235245865324252 }, "scio.prompt2_mqa_single_correct_sk:che|0": { "acc": 0.18699186991869918, "acc_stderr": 0.024910116828719235, "em_relaxed": 0.1910569105691057, "em_relaxed_stderr": 0.025116395774587044 }, "skj1.prompt2_skj1_sk|0": { "acc": 0.63, "acc_stderr": 0.04852365870939099, "em_relaxed": 0.64, "em_relaxed_stderr": 0.048241815132442176 }, "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { "acc": 0.10282776349614396, "acc_stderr": 0.010896388982494143, "em_relaxed": 0.12853470437017994, "em_relaxed_stderr": 0.012006729887990094 }, "e_tests.prompt3_qa_sk:informatika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.006923076923076923, "qf1_stderr": 0.006424094288133824, "rouge1": 0.00046479944765467505, "rouge1_stderr": 0.0001876188412456943, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.00046479944765467505, "rougeL_stderr": 0.0001876188412456943 }, "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { "acc": 0.23026315789473684, "acc_stderr": 0.03426059424403165, "em_relaxed": 0.23026315789473684, "em_relaxed_stderr": 0.03426059424403165 }, "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { "acc": 0.27, "acc_stderr": 0.04461960433384741, "em_relaxed": 0.28, "em_relaxed_stderr": 0.04512608598542127 }, "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { "acc": 0.2145214521452145, "acc_stderr": 0.023621042274383643, "em_relaxed": 0.24092409240924093, "em_relaxed_stderr": 0.0246081660183802 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { "acc": 0.2676767676767677, "acc_stderr": 0.03154449888270286, "em_relaxed": 0.2676767676767677, "em_relaxed_stderr": 0.03154449888270286 }, "financial_exams.prompt3_completion_sk|0": { "acc": 0.13343328335832083, "acc_stderr": 0.009313609263721821, "em_relaxed": 0.13343328335832083, "em_relaxed_stderr": 0.009313609263721821 }, "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { "acc": 0.19289340101522842, "acc_stderr": 0.028183576131702342, "em_relaxed": 0.19289340101522842, "em_relaxed_stderr": 0.028183576131702342 }, "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { "acc": 0.26900584795321636, "acc_stderr": 0.0340105262010409, "em_relaxed": 0.2807017543859649, "em_relaxed_stderr": 0.034462962170884265 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { "acc": 0.15555555555555556, "acc_stderr": 0.03130948364878314, "em_relaxed": 0.15555555555555556, "em_relaxed_stderr": 0.03130948364878314 }, "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { "acc": 0.18, "acc_stderr": 0.03861229196653695, "em_relaxed": 0.18, "em_relaxed_stderr": 0.03861229196653695 }, "scio.prompt3_mqa_single_correct_sk:aj|0": { "acc": 0.5, "acc_stderr": 0.16666666666666666, "em_relaxed": 0.5, "em_relaxed_stderr": 0.16666666666666666 }, "e_tests.prompt3_mqa_single_correct_sk:geografia|0": { "acc": 0.22448979591836735, "acc_stderr": 0.06022425581505364, "em_relaxed": 0.22448979591836735, "em_relaxed_stderr": 0.06022425581505364 }, "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { "acc": 0.18181818181818182, "acc_stderr": 0.03896103896103899, "em_relaxed": 0.18181818181818182, "em_relaxed_stderr": 0.03896103896103899 }, "e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { "acc": 0.15228426395939088, "acc_stderr": 0.02566401503870399, "em_relaxed": 0.15228426395939088, "em_relaxed_stderr": 0.02566401503870399 }, "e_tests.prompt1_qa_sk:obcianska-nauka|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "e_tests.prompt3_mqa_single_correct_sk:matematika|0": { "acc": 0.05128205128205128, "acc_stderr": 0.035781575124326184, "em_relaxed": 0.23076923076923078, "em_relaxed_stderr": 0.0683479474337414 }, "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { "acc": 0.26515151515151514, "acc_stderr": 0.0272187441820065, "em_relaxed": 0.2765151515151515, "em_relaxed_stderr": 0.027580130232496353 }, "e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { "acc": 0.2, "acc_stderr": 0.04020151261036846, "em_relaxed": 0.22, "em_relaxed_stderr": 0.04163331998932268 }, "mmlu.prompt2_mqa_single_correct_sk:management|0": { "acc": 0.19, "acc_stderr": 0.03942772444036622, "em_relaxed": 0.19, "em_relaxed_stderr": 0.03942772444036622 }, "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { "acc": 0.2206896551724138, "acc_stderr": 0.03455930201924812, "em_relaxed": 0.2413793103448276, "em_relaxed_stderr": 0.03565998174135302 }, "sklep.prompt1_nli_3_classes_sk:nli|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { "acc": 0.18468468468468469, "acc_stderr": 0.026102500004017645, "em_relaxed": 0.1981981981981982, "em_relaxed_stderr": 0.026815582053277203 }, "sklep.prompt3_nli_binary_sk:rte|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { "acc": 0.18624161073825504, "acc_stderr": 0.015959806298600995, "em_relaxed": 0.1912751677852349, "em_relaxed_stderr": 0.01612394106868719 }, "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { "acc": 0.22772277227722773, "acc_stderr": 0.024131614233378947, "em_relaxed": 0.264026402640264, "em_relaxed_stderr": 0.0253659612714007 }, "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { "acc": 0.19, "acc_stderr": 0.039427724440366234, "em_relaxed": 0.19, "em_relaxed_stderr": 0.039427724440366234 }, "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { "acc": 0.21, "acc_stderr": 0.04093601807403326, "em_relaxed": 0.22, "em_relaxed_stderr": 0.04163331998932268 }, "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { "acc": 0.24646983311938384, "acc_stderr": 0.015450495390660532, "em_relaxed": 0.24775353016688062, "em_relaxed_stderr": 0.015477478391132707 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { "acc": 0.31666666666666665, "acc_stderr": 0.034768900963930385, "em_relaxed": 0.31666666666666665, "em_relaxed_stderr": 0.034768900963930385 }, "arc.prompt2_mqa_single_correct_sk|0": { "acc": 0.23353293413173654, "acc_stderr": 0.012379394294094546, "em_relaxed": 0.23353293413173654, "em_relaxed_stderr": 0.012379394294094546 }, "slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": { "acc": 0.14746543778801843, "acc_stderr": 0.024125384514879104, "em_relaxed": 0.14746543778801843, "em_relaxed_stderr": 0.024125384514879104 }, "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { "acc": 0.22, "acc_stderr": 0.04163331998932269, "em_relaxed": 0.22, "em_relaxed_stderr": 0.04163331998932269 }, "slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": { "acc": 0.11428571428571428, "acc_stderr": 0.031197955091381097, "em_relaxed": 0.11428571428571428, "em_relaxed_stderr": 0.031197955091381097 }, "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { "acc": 0.2359249329758713, "acc_stderr": 0.02201321444385349, "em_relaxed": 0.2359249329758713, "em_relaxed_stderr": 0.02201321444385349 }, "e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": { "acc": 0.0, "acc_stderr": 0.0 }, "e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": { "acc": 0.017857142857142856, "acc_stderr": 0.017857142857142867 }, "toxic_sk.prompt2_toxicity_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "e_tests.prompt2_mqa_single_correct_sk:fyzika|0": { "acc": 0.13636363636363635, "acc_stderr": 0.07488677009526491, "em_relaxed": 0.22727272727272727, "em_relaxed_stderr": 0.09144861547306321 }, "senti_sk.prompt1_sentiment_binary_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { "acc": 0.25288831835686776, "acc_stderr": 0.015583584105316876, "em_relaxed": 0.25417201540436457, "em_relaxed_stderr": 0.01560965861721762 }, "slovak_tests.prompt2_qa_sk:monitor9|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { "acc": 0.23049645390070922, "acc_stderr": 0.025123739226872395, "em_relaxed": 0.23404255319148937, "em_relaxed_stderr": 0.025257861359432386 }, "e_tests.prompt1_mqa_single_correct_sk:informatika|0": { "acc": 0.264, "acc_stderr": 0.03958494337416042, "em_relaxed": 0.272, "em_relaxed_stderr": 0.0399612715739433 }, "e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": { "acc": 0.02142857142857143, "acc_stderr": 0.012282463299882668 }, "e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { "acc": 0.2349799732977303, "acc_stderr": 0.015502474593163047, "em_relaxed": 0.2349799732977303, "em_relaxed_stderr": 0.015502474593163047 }, "e_tests.prompt1_qa_sk:chemia|0": { "qem": 0.018518518518518517, "qem_stderr": 0.018518518518518517, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.00026455026455026457, "rouge1_stderr": 0.00026455026455026484, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.00026455026455026457, "rougeL_stderr": 0.00026455026455026484 }, "scio.prompt2_mqa_single_correct_sk:aj|0": { "acc": 0.6, "acc_stderr": 0.16329931618554522, "em_relaxed": 0.6, "em_relaxed_stderr": 0.16329931618554522 }, "e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": { "acc": 0.0, "acc_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { "acc": 0.19638242894056848, "acc_stderr": 0.020220056896337255, "em_relaxed": 0.20155038759689922, "em_relaxed_stderr": 0.020418410166350903 }, "slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": { "acc": 0.15176151761517614, "acc_stderr": 0.018703199518943774, "em_relaxed": 0.15447154471544716, "em_relaxed_stderr": 0.01883928643967625 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { "acc": 0.37373737373737376, "acc_stderr": 0.04887069039502487, "em_relaxed": 0.3838383838383838, "em_relaxed_stderr": 0.04912566964083466 }, "e_tests.prompt2_qa_sk:biologia|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { "acc": 0.18, "acc_stderr": 0.03861229196653697, "em_relaxed": 0.18, "em_relaxed_stderr": 0.03861229196653697 }, "e_tests.prompt1_qa_sk:informatika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.00824175824175824, "qf1_stderr": 0.006643857376866051, "rouge1": 0.00033664858861620243, "rouge1_stderr": 0.00019799367371799002, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.00033664858861620243, "rougeL_stderr": 0.00019799367371799002 }, "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { "acc": 0.22784810126582278, "acc_stderr": 0.03347528264060925, "em_relaxed": 0.23417721518987342, "em_relaxed_stderr": 0.03379766010403987 }, "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "e_tests.prompt2_mqa_single_correct_sk:informatika|0": { "acc": 0.224, "acc_stderr": 0.037440705810827146, "em_relaxed": 0.232, "em_relaxed_stderr": 0.03790650637827826 }, "e_tests.prompt2_qa_sk:obcianska-nauka|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "mmlu.prompt1_mqa_single_correct_sk:management|0": { "acc": 0.21, "acc_stderr": 0.040936018074033256, "em_relaxed": 0.21, "em_relaxed_stderr": 0.040936018074033256 }, "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { "acc": 0.23728813559322035, "acc_stderr": 0.03933012549934385, "em_relaxed": 0.23728813559322035, "em_relaxed_stderr": 0.03933012549934385 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { "acc": 0.2488479262672811, "acc_stderr": 0.029417385564063876, "em_relaxed": 0.2626728110599078, "em_relaxed_stderr": 0.029944068647646538 }, "e_tests.prompt3_qa_sk:fyzika|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0014626760356334343, "rouge1_stderr": 0.0010249453223271077, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0014626760356334343, "rougeL_stderr": 0.0010249453223271077 }, "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { "acc": 0.13247863247863248, "acc_stderr": 0.022209309073165606, "em_relaxed": 0.13247863247863248, "em_relaxed_stderr": 0.022209309073165606 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { "acc": 0.19172932330827067, "acc_stderr": 0.024182419147367247, "em_relaxed": 0.19924812030075187, "em_relaxed_stderr": 0.02453709604824174 }, "slovak_tests.prompt1_qa_sk:maturita|0": { "qem": 0.01327433628318584, "qem_stderr": 0.007629803632081149, "qf1": 0.00454104547024901, "qf1_stderr": 0.0026612932202592188, "rouge1": 0.001362195307144739, "rouge1_stderr": 0.0004933551702641079, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0012709627553702657, "rougeL_stderr": 0.0004501834684090662 }, "e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": { "acc": 0.0, "acc_stderr": 0.0 }, "senti_sk.prompt3_sentiment_3_class_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { "acc": 0.1919191919191919, "acc_stderr": 0.028057791672989017, "em_relaxed": 0.1919191919191919, "em_relaxed_stderr": 0.028057791672989017 }, "e_tests.prompt1_qa_sk:dejepis|0": { "qem": 0.0, "qem_stderr": 0.0, "qf1": 0.0, "qf1_stderr": 0.0, "rouge1": 0.0, "rouge1_stderr": 0.0, "rouge2": 0.0, "rouge2_stderr": 0.0, "rougeL": 0.0, "rougeL_stderr": 0.0 }, "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": { "acc": 0.017857142857142856, "acc_stderr": 0.017857142857142867 }, "senti_sk.prompt1_sentiment_3_class_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { "acc": 0.2323943661971831, "acc_stderr": 0.03556905428023101, "em_relaxed": 0.2323943661971831, "em_relaxed_stderr": 0.03556905428023101 }, "slohry.prompt2_identity_form_sk:identity_form_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_declension_sk:declension_mid|0": { "acc": 0.011976047904191617, "acc_stderr": 0.005960989845848317, "em_norm_relaxed": 0.011976047904191617, "em_norm_relaxed_stderr": 0.005960989845848317 }, "slohry.prompt1_declension_sk:declension_low|0": { "acc": 0.0029940119760479044, "acc_stderr": 0.0029940119760478844, "em_norm_relaxed": 0.0029940119760479044, "em_norm_relaxed_stderr": 0.0029940119760478844 }, "analogies_sk.prompt1_analogy_sk|0": { "acc": 0.0, "acc_stderr": 0.0 }, "slovak_trivia.prompt3_mqa_single_correct_sk|0": { "acc": 0.22096317280453256, "acc_stderr": 0.022114015575065166, "em_relaxed": 0.2237960339943343, "em_relaxed_stderr": 0.02221481955471835 }, "slohry.prompt3_identity_form_sk:identity_form_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_gradation_sk:gradation_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_inflection_sk:inflection_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_gradation_sk:gradation_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "chiby.prompt2_chiby_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_inflection_sk:inflection_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slovak_trivia.prompt2_mqa_single_correct_sk|0": { "acc": 0.1501416430594901, "acc_stderr": 0.01903936786327736, "em_relaxed": 0.1558073654390935, "em_relaxed_stderr": 0.019330515671326005 }, "chiby.prompt3_chiby_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_gradation_sk:gradation_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_declension_sk:declension_high|0": { "acc": 0.017964071856287425, "acc_stderr": 0.0072785348116838205, "em_norm_relaxed": 0.017964071856287425, "em_norm_relaxed_stderr": 0.0072785348116838205 }, "slohry.prompt2_inflection_sk:inflection_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_declension_sk:declension_low|0": { "acc": 0.0029940119760479044, "acc_stderr": 0.0029940119760478896, "em_norm_relaxed": 0.0029940119760479044, "em_norm_relaxed_stderr": 0.0029940119760478896 }, "slohry.prompt3_gradation_sk:gradation_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_gradation_sk:gradation_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "chiby.prompt1_chiby_sk|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_declension_sk:declension_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_identity_form_sk:identity_form_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_identity_form_sk:identity_form_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_declension_sk:declension_high|0": { "acc": 0.011976047904191617, "acc_stderr": 0.005960989845848319, "em_norm_relaxed": 0.011976047904191617, "em_norm_relaxed_stderr": 0.005960989845848319 }, "slohry.prompt3_declension_sk:declension_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_inflection_sk:inflection_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slovak_trivia.prompt1_mqa_single_correct_sk|0": { "acc": 0.22662889518413598, "acc_stderr": 0.02231414649207437, "em_relaxed": 0.23229461756373937, "em_relaxed_stderr": 0.022508447165267422 }, "slohry.prompt1_identity_form_sk:identity_form_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_declension_sk:declension_high|0": { "acc": 0.020958083832335328, "acc_stderr": 0.007849722658097238, "em_norm_relaxed": 0.020958083832335328, "em_norm_relaxed_stderr": 0.007849722658097238 }, "slohry.prompt2_inflection_sk:inflection_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_declension_sk:declension_mid|0": { "acc": 0.014970059880239521, "acc_stderr": 0.006654483730236385, "em_norm_relaxed": 0.014970059880239521, "em_norm_relaxed_stderr": 0.006654483730236385 }, "analogies_sk.prompt2_analogy_sk|0": { "acc": 0.0, "acc_stderr": 0.0 }, "slohry.prompt1_gradation_sk:gradation_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_gradation_sk:gradation_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_inflection_sk:inflection_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_identity_form_sk:identity_form_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_gradation_sk:gradation_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "analogies_sk.prompt3_analogy_sk|0": { "acc": 0.0, "acc_stderr": 0.0 }, "slohry.prompt1_identity_form_sk:identity_form_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_identity_form_sk:identity_form_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_inflection_sk:inflection_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt3_inflection_sk:inflection_low|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_gradation_sk:gradation_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt1_inflection_sk:inflection_mid|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "slohry.prompt2_identity_form_sk:identity_form_high|0": { "acc": 0.0, "acc_stderr": 0.0, "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0 }, "e_tests_mqa_SK_AVERAGE|0": { "em_relaxed": 0.21755858691850297, "acc": 0.15008726003490402, "em_relaxed_stderr": 0.03505820050399378, "acc_stderr": 0.027287029108301525, "em_relaxed__prompts": { "1": 0.23819517313746066, "2": 0.2088142707240294, "3": 0.20566631689401887 }, "acc__prompts": { "1": 0.16454749439042632, "2": 0.14360508601346297, "3": 0.14210919970082272 } }, "e_tests_mqa_AVERAGE|0": { "em_relaxed": 0.21755858691850297, "em_relaxed_stderr": 0.03505820050399378, "acc": 0.15008726003490402, "acc_stderr": 0.027287029108301525 }, "mmlu_SK_AVERAGE|0": { "em_relaxed": 0.22816822334507222, "acc": 0.2224518960853366, "em_relaxed_stderr": 0.025975113405158696, "acc_stderr": 0.02571374453987217, "em_relaxed__prompts": { "1": 0.24628693921298422, "2": 0.21719491655182974, "3": 0.22102281427040268 }, "acc__prompts": { "1": 0.2399326290001531, "2": 0.21137651201959884, "3": 0.21604654723625785 } }, "mmlu_AVERAGE|0": { "em_relaxed": 0.22816822334507222, "em_relaxed_stderr": 0.025975113405158696, "acc": 0.2224518960853366, "acc_stderr": 0.02571374453987217 }, "scio_SK_AVERAGE|0": { "em_relaxed": 0.24911297852474323, "acc": 0.238468720821662, "em_relaxed_stderr": 0.020694508860887174, "acc_stderr": 0.0204222231602884, "em_relaxed__prompts": { "1": 0.261624649859944, "3": 0.23865546218487396, "2": 0.24705882352941178 }, "acc__prompts": { "1": 0.2504201680672269, "3": 0.2280112044817927, "2": 0.23697478991596638 } }, "scio_AVERAGE|0": { "em_relaxed": 0.24911297852474323, "em_relaxed_stderr": 0.020694508860887174, "acc": 0.238468720821662, "acc_stderr": 0.0204222231602884 }, "slovak_tests_qa_SK_AVERAGE|0": { "rouge2": 0.0, "rouge1": 0.0011809129165421361, "rougeL": 0.0011384377140699951, "qf1": 0.003009101538513303, "qem": 0.012605042016806723, "rouge2_stderr": 0.0, "rouge1_stderr": 0.0007262749376038493, "rougeL_stderr": 0.0006990122730613884, "qf1_stderr": 0.0019761043274910683, "qem_stderr": 0.0072451076506316785, "rouge2__prompts": { "3": 0.0, "2": 0.0, "1": 0.0 }, "rouge1__prompts": { "3": 0.001309304483429979, "2": 0.0008896015463691078, "1": 0.001343832719827322 }, "rougeL__prompts": { "3": 0.001309304483429979, "2": 0.0008488085301334873, "1": 0.0012572001286465196 }, "qf1__prompts": { "3": 0.0021008403361344537, "2": 0.0026143790849673205, "1": 0.004312085194438136 }, "qem__prompts": { "3": 0.012605042016806723, "2": 0.012605042016806723, "1": 0.012605042016806723 } }, "slovak_tests_qa_AVERAGE|0": { "rouge2": 0.0, "rouge2_stderr": 0.0, "rouge1": 0.0011809129165421361, "rouge1_stderr": 0.0007262749376038493, "rougeL": 0.0011384377140699951, "rougeL_stderr": 0.0006990122730613884, "qf1": 0.003009101538513303, "qf1_stderr": 0.0019761043274910683, "qem": 0.012605042016806723, "qem_stderr": 0.0072451076506316785 }, "slovak_tests_mqa_SK_AVERAGE|0": { "em_relaxed": 0.17543859649122806, "acc": 0.17389966143428748, "em_relaxed_stderr": 0.02232972578404266, "acc_stderr": 0.02225738292492629, "em_relaxed__prompts": { "1": 0.20498614958448755, "3": 0.17820867959372114, "2": 0.14312096029547552 }, "acc__prompts": { "1": 0.20221606648199447, "3": 0.17820867959372114, "2": 0.14127423822714683 } }, "slovak_tests_mqa_AVERAGE|0": { "em_relaxed": 0.17543859649122806, "em_relaxed_stderr": 0.02232972578404266, "acc": 0.17389966143428748, "acc_stderr": 0.02225738292492629 }, "movie_review_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "2": 0.0, "3": 0.0, "1": 0.0 }, "acc__prompts": { "2": 0.0, "3": 0.0, "1": 0.0 } }, "movie_review_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "e_tests_qa_SK_AVERAGE|0": { "rouge2": 0.0, "rouge1": 0.00029664081444656757, "rougeL": 0.00029664081444656757, "qf1": 0.0026099107814683436, "qem": 0.002257336343115124, "rouge2_stderr": 0.0, "rouge1_stderr": 0.0001935712320593315, "rougeL_stderr": 0.0001935712320593315, "qf1_stderr": 0.002326519969477271, "qem_stderr": 0.002257336343115124, "rouge2__prompts": { "1": 0.0, "2": 0.0, "3": 0.0 }, "rouge1__prompts": { "1": 0.00011697710542177868, "2": 0.0002733713466381956, "3": 0.0004995739912797284 }, "rougeL__prompts": { "1": 0.00011697710542177868, "2": 0.0002733713466381956, "3": 0.0004995739912797284 }, "qf1__prompts": { "1": 0.0025798129635601417, "2": 0.00290228958400516, "3": 0.002347629796839729 }, "qem__prompts": { "1": 0.002257336343115124, "2": 0.002257336343115124, "3": 0.002257336343115124 } }, "e_tests_qa_AVERAGE|0": { "rouge2": 0.0, "rouge2_stderr": 0.0, "rouge1": 0.00029664081444656757, "rouge1_stderr": 0.0001935712320593315, "rougeL": 0.00029664081444656757, "rougeL_stderr": 0.0001935712320593315, "qf1": 0.0026099107814683436, "qf1_stderr": 0.002326519969477271, "qem": 0.002257336343115124, "qem_stderr": 0.002257336343115124 }, "sklep_nli_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "3": 0.0, "2": 0.0, "1": 0.0 }, "acc__prompts": { "3": 0.0, "2": 0.0, "1": 0.0 } }, "sklep_nli_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "sklep_sentiment_analysis_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "2": 0.0, "1": 0.0, "3": 0.0 }, "acc__prompts": { "2": 0.0, "1": 0.0, "3": 0.0 } }, "sklep_sentiment_analysis_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "arc_SK_AVERAGE|0": { "em_relaxed": 0.24493869404049046, "acc": 0.24493869404049046, "em_relaxed_stderr": 0.012565550837641016, "acc_stderr": 0.012565550837641016, "em_relaxed__prompts": { "1": 0.2728828058169376, "3": 0.22840034217279725, "2": 0.23353293413173654 }, "acc__prompts": { "1": 0.2728828058169376, "3": 0.22840034217279725, "2": 0.23353293413173654 } }, "arc_AVERAGE|0": { "em_relaxed": 0.24493869404049046, "em_relaxed_stderr": 0.012565550837641016, "acc": 0.24493869404049046, "acc_stderr": 0.012565550837641016 }, "belebele_SK_AVERAGE|0": { "em_relaxed": 0.2611111111111111, "acc": 0.2537037037037037, "em_relaxed_stderr": 0.014646428295956382, "acc_stderr": 0.014509896842640889, "em_relaxed__prompts": { "2": 0.25666666666666665, "3": 0.2544444444444444, "1": 0.2722222222222222 }, "acc__prompts": { "2": 0.2511111111111111, "3": 0.24666666666666667, "1": 0.2633333333333333 } }, "belebele_AVERAGE|0": { "em_relaxed": 0.2611111111111111, "em_relaxed_stderr": 0.014646428295956382, "acc": 0.2537037037037037, "acc_stderr": 0.014509896842640889 }, "senti_sk_3class_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "2": 0.0, "3": 0.0, "1": 0.0 }, "acc__prompts": { "2": 0.0, "3": 0.0, "1": 0.0 } }, "senti_sk_3class_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "sklep_hate_speech_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "2": 0.0, "1": 0.0, "3": 0.0 }, "acc__prompts": { "2": 0.0, "1": 0.0, "3": 0.0 } }, "sklep_hate_speech_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "slovakcopa_SK_AVERAGE|0": { "em_relaxed": 0.05333333333333334, "acc": 0.05333333333333334, "em_relaxed_stderr": 0.007876087666852084, "acc_stderr": 0.007876087666852084, "em_relaxed__prompts": { "3": 0.144, "2": 0.006, "1": 0.01 }, "acc__prompts": { "3": 0.144, "2": 0.006, "1": 0.01 } }, "slovakcopa_AVERAGE|0": { "em_relaxed": 0.05333333333333334, "em_relaxed_stderr": 0.007876087666852084, "acc": 0.05333333333333334, "acc_stderr": 0.007876087666852084 }, "globalmmlu_lite_SK_AVERAGE|0": { "em_relaxed": 0.21333333333333335, "acc": 0.21083333333333334, "em_relaxed_stderr": 0.020468272436489743, "acc_stderr": 0.020379605193633, "em_relaxed__prompts": { "1": 0.2425, "3": 0.205, "2": 0.1925 }, "acc__prompts": { "1": 0.24, "3": 0.2025, "2": 0.19 } }, "globalmmlu_lite_AVERAGE|0": { "em_relaxed": 0.21333333333333335, "em_relaxed_stderr": 0.020468272436489743, "acc": 0.21083333333333334, "acc_stderr": 0.020379605193633 }, "sklep_qa_SK_AVERAGE|0": { "qf1": 0.0070376260418807705, "qem": 0.00027827054854081884, "qf1_stderr": 0.0003824771262616922, "qem_stderr": 0.0001679351079586058, "qf1__prompts": { "3": 0.006093522810123945, "2": 0.006743145308527306, "1": 0.008276210006991061 }, "qem__prompts": { "3": 0.0002087029114056141, "2": 0.0002087029114056141, "1": 0.0004174058228112282 } }, "sklep_qa_AVERAGE|0": { "qf1": 0.0070376260418807705, "qf1_stderr": 0.0003824771262616922, "qem": 0.00027827054854081884, "qem_stderr": 0.0001679351079586058 }, "financial_exams_SK_AVERAGE|0": { "em_relaxed": 0.08995502248875563, "acc": 0.0887056471764118, "em_relaxed_stderr": 0.007689948855002123, "acc_stderr": 0.007626104510560348, "em_relaxed__prompts": { "1": 0.05697151424287856, "2": 0.07946026986506746, "3": 0.13343328335832083 }, "acc__prompts": { "1": 0.053973013493253376, "2": 0.07871064467766117, "3": 0.13343328335832083 } }, "financial_exams_AVERAGE|0": { "em_relaxed": 0.08995502248875563, "em_relaxed_stderr": 0.007689948855002123, "acc": 0.0887056471764118, "acc_stderr": 0.007626104510560348 }, "ifeval_sk_SK_AVERAGE|0": { "prompt_level_strict_acc": 0.0782122905027933, "prompt_level_loose_acc": 0.0782122905027933, "inst_level_strict_acc": 0.17312348668280872, "inst_level_loose_acc": 0.17312348668280872, "prompt_level_strict_acc_stderr": 0.011597665111537078, "prompt_level_loose_acc_stderr": 0.011597665111537078, "inst_level_strict_acc_stderr": 0.0004210039776560394, "inst_level_loose_acc_stderr": 0.0004210039776560394 }, "ifeval_sk_AVERAGE|0": { "prompt_level_strict_acc": 0.0782122905027933, "prompt_level_strict_acc_stderr": 0.011597665111537078, "prompt_level_loose_acc": 0.0782122905027933, "prompt_level_loose_acc_stderr": 0.011597665111537078, "inst_level_strict_acc": 0.17312348668280872, "inst_level_strict_acc_stderr": 0.0004210039776560394, "inst_level_loose_acc": 0.17312348668280872, "inst_level_loose_acc_stderr": 0.0004210039776560394 }, "toxic_sk_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "1": 0.0, "3": 0.0, "2": 0.0 }, "acc__prompts": { "1": 0.0, "3": 0.0, "2": 0.0 } }, "toxic_sk_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "driving_test_SK_AVERAGE|0": { "em_relaxed": 0.1407513076557299, "acc": 0.13742272943414172, "em_relaxed_stderr": 0.013052324431028241, "acc_stderr": 0.012921192285229888, "em_relaxed__prompts": { "2": 0.10271041369472182, "1": 0.16833095577746077, "3": 0.15121255349500715 }, "acc__prompts": { "2": 0.09985734664764621, "1": 0.16405135520684735, "3": 0.14835948644793154 } }, "driving_test_AVERAGE|0": { "em_relaxed": 0.1407513076557299, "em_relaxed_stderr": 0.013052324431028241, "acc": 0.13742272943414172, "acc_stderr": 0.012921192285229888 }, "sklep_rte_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "1": 0.0, "2": 0.0, "3": 0.0 }, "acc__prompts": { "1": 0.0, "2": 0.0, "3": 0.0 } }, "sklep_rte_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "multi_wiki_qa_SK_AVERAGE|0": { "qf1": 0.005263913929238519, "qem": 0.0005336535254486025, "qf1_stderr": 0.0005228307644937593, "qem_stderr": 0.00032536056243172176, "qf1__prompts": { "3": 0.005765729879562343, "2": 0.005417769336679904, "1": 0.004608242571473309 }, "qem__prompts": { "3": 0.0006003602161296778, "2": 0.0006003602161296778, "1": 0.00040024014408645187 } }, "multi_wiki_qa_AVERAGE|0": { "qf1": 0.005263913929238519, "qf1_stderr": 0.0005228307644937593, "qem": 0.0005336535254486025, "qem_stderr": 0.00032536056243172176 }, "truthful_qa_SK_AVERAGE|0": { "em_relaxed": 0.09682947729220223, "acc": 0.06469580119965725, "em_relaxed_stderr": 0.010525690171591523, "acc_stderr": 0.008619652748939895, "em_relaxed__prompts": { "2": 0.08354755784061697, "1": 0.07840616966580977, "3": 0.12853470437017994 }, "acc__prompts": { "2": 0.04884318766066838, "1": 0.042416452442159386, "3": 0.10282776349614396 } }, "truthful_qa_AVERAGE|0": { "em_relaxed": 0.09682947729220223, "em_relaxed_stderr": 0.010525690171591523, "acc": 0.06469580119965725, "acc_stderr": 0.008619652748939895 }, "senti_sk_binary_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "3": 0.0, "2": 0.0, "1": 0.0 }, "acc__prompts": { "3": 0.0, "2": 0.0, "1": 0.0 } }, "senti_sk_binary_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "skj1_SK_AVERAGE|0": { "em_relaxed": 0.61, "acc": 0.5966666666666667, "em_relaxed_stderr": 0.04721666475945084, "acc_stderr": 0.04715805991356215, "em_relaxed__prompts": { "1": 0.44, "3": 0.75, "2": 0.64 }, "acc__prompts": { "1": 0.41, "3": 0.75, "2": 0.63 } }, "skj1_AVERAGE|0": { "em_relaxed": 0.61, "em_relaxed_stderr": 0.04721666475945084, "acc": 0.5966666666666667, "acc_stderr": 0.04715805991356215 }, "slohry_SK_AVERAGE|0": { "em_norm_relaxed": 0.002331002331002331, "acc": 0.002331002331002331, "em_norm_relaxed_stderr": 0.0011036777204322752, "acc_stderr": 0.0011036777204322752, "em_norm_relaxed__prompts": { "2": 0.003246753246753247, "1": 0.0027472527472527475, "3": 0.000999000999000999 }, "acc__prompts": { "2": 0.003246753246753247, "1": 0.0027472527472527475, "3": 0.000999000999000999 } }, "slohry_AVERAGE|0": { "em_norm_relaxed": 0.002331002331002331, "em_norm_relaxed_stderr": 0.0011036777204322752, "acc": 0.002331002331002331, "acc_stderr": 0.0011036777204322752 }, "analogies_sk_SK_AVERAGE|0": { "acc": 0.0, "acc_stderr": 0.0, "acc__prompts": { "1": 0.0, "2": 0.0, "3": 0.0 } }, "analogies_sk_AVERAGE|0": { "acc": 0.0, "acc_stderr": 0.0 }, "slovak_trivia_SK_AVERAGE|0": { "em_relaxed": 0.20396600566572237, "acc": 0.19924457034938622, "em_relaxed_stderr": 0.021351260797103925, "acc_stderr": 0.021155843310138964, "em_relaxed__prompts": { "3": 0.2237960339943343, "2": 0.1558073654390935, "1": 0.23229461756373937 }, "acc__prompts": { "3": 0.22096317280453256, "2": 0.1501416430594901, "1": 0.22662889518413598 } }, "slovak_trivia_AVERAGE|0": { "em_relaxed": 0.20396600566572237, "em_relaxed_stderr": 0.021351260797103925, "acc": 0.19924457034938622, "acc_stderr": 0.021155843310138964 }, "chiby_SK_AVERAGE|0": { "em_norm_relaxed": 0.0, "acc": 0.0, "em_norm_relaxed_stderr": 0.0, "acc_stderr": 0.0, "em_norm_relaxed__prompts": { "2": 0.0, "3": 0.0, "1": 0.0 }, "acc__prompts": { "2": 0.0, "3": 0.0, "1": 0.0 } }, "chiby_AVERAGE|0": { "em_norm_relaxed": 0.0, "em_norm_relaxed_stderr": 0.0, "acc": 0.0, "acc_stderr": 0.0 }, "SK_AVERAGE|0": { "em_relaxed": 0.21419205155386342, "acc": 0.10986595916434654, "rouge2": 0.0, "rouge1": 0.0007387768654943519, "rougeL": 0.0007175392642582814, "qf1": 0.004480138072775234, "qem": 0.003918575608477817, "em_norm_relaxed": 0.0002331002331002331, "prompt_level_strict_acc": 0.0782122905027933, "prompt_level_loose_acc": 0.0782122905027933, "inst_level_strict_acc": 0.17312348668280872, "inst_level_loose_acc": 0.17312348668280872, "em_relaxed_stderr": 0.006300951163940428, "acc_stderr": 0.003263889036835735, "rouge2_stderr": 0.0, "rouge1_stderr": 0.00037581415183320424, "rougeL_stderr": 0.0003626596130572774, "qf1_stderr": 0.0007801171483309406, "qem_stderr": 0.0018993619641089511, "em_norm_relaxed_stderr": 0.00011036777204322752, "prompt_level_strict_acc_stderr": 0.011597665111537078, "prompt_level_loose_acc_stderr": 0.011597665111537078, "inst_level_strict_acc_stderr": 0.0004210039776560394, "inst_level_loose_acc_stderr": 0.0004210039776560394 }, "AVERAGE|0": { "em_relaxed": 0.18120805585001873, "acc": 0.09273256151262048, "rouge2": 0.0, "rouge1": 0.0007387768654943519, "rougeL": 0.0007175392642582814, "qf1": 0.004480138072775234, "qem": 0.003918575608477817, "em_norm_relaxed": 0.000259000259000259, "prompt_level_strict_acc": 0.0782122905027933, "prompt_level_loose_acc": 0.0782122905027933, "inst_level_strict_acc": 0.17312348668280872, "inst_level_loose_acc": 0.17312348668280872, "em_relaxed_stderr": 0.005577871787987613, "acc_stderr": 0.002843082460505095, "rouge2_stderr": 0.0, "rouge1_stderr": 0.00037581415183320424, "rougeL_stderr": 0.0003626596130572774, "qf1_stderr": 0.0007801171483309406, "qem_stderr": 0.0018993619641089511, "em_norm_relaxed_stderr": 0.00012263085782580836, "prompt_level_strict_acc_stderr": 0.011597665111537078, "prompt_level_loose_acc_stderr": 0.011597665111537078, "inst_level_strict_acc_stderr": 0.0004210039776560394, "inst_level_loose_acc_stderr": 0.0004210039776560394 } } }