From 5b003d87278a44350a2ed5cb61d2098579f0032d Mon Sep 17 00:00:00 2001 From: Jakub Date: Sun, 25 Oct 2026 01:03:33 +0200 Subject: [PATCH] delete --- .../lmeval/comparison_with_100k.csv | 12 - .../configs/dataset_access_report.json | 622 - .../skbench/configs/export_lr_0p00010.yaml | 12 - .../skbench/configs/model_config.json | 59 - .../skbench/configs/omitted_tasks.txt | 190 - .../skbench/configs/task_configs.json | 13475 ---------------- .../benchmarks/skbench/configs/tasks.txt | 222 - .../skbench/configs/tasks_sk_zero.txt | 412 - .../configs/tasks_sk_zero_available.txt | 225 - .../tasks_sk_zero_available_without_nli.txt | 222 - .../configs/tasks_sk_zero_unavailable.txt | 187 - 11 files changed, 15638 deletions(-) delete mode 100644 aya_300k_lr_experiments/benchmarks/lmeval/comparison_with_100k.csv delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/dataset_access_report.json delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/export_lr_0p00010.yaml delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/model_config.json delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/omitted_tasks.txt delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/task_configs.json delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/tasks.txt delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero.txt delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available.txt delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available_without_nli.txt delete mode 100644 aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_unavailable.txt diff --git a/aya_300k_lr_experiments/benchmarks/lmeval/comparison_with_100k.csv b/aya_300k_lr_experiments/benchmarks/lmeval/comparison_with_100k.csv deleted file mode 100644 index 3376c2b..0000000 --- a/aya_300k_lr_experiments/benchmarks/lmeval/comparison_with_100k.csv +++ /dev/null @@ -1,12 +0,0 @@ -model_id,training_samples,model_name,source_result_json,arc_acc_norm,hellaswag_acc_norm,mmlu_acc,truthfulqa_mc1_acc,truthfulqa_mc2_acc,sklegal_acc_norm,skquad_f1 -01_baseline_r16_a16_lr5e5_adamw,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_01_baseline_r16_a16_lr5e5_adamw,lmeval/mistral_aya(hruska)/01_baseline_r16_a16_lr5e5_adamw/results_2026-09-30T21-23-33.435952.json,0.34046193327630453,0.5209277807063785,0.3549992344204563,0.17994858611825193,0.34391089322085056,0.46851574212893554,36.6797346829585 -02_optimizer_adamw8bit_r16_a16_lr5e5,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_02_optimizer_adamw8bit_r16_a16_lr5e5,lmeval/mistral_aya(hruska)/02_optimizer_adamw8bit_r16_a16_lr5e5/results_2026-10-01T04-23-23.791749.json,0.3378956372968349,0.521244069583553,0.35438677078548464,0.1928020565552699,0.35373417730948353,0.4715142428785607,36.21782844597198 -03_rank8_a16_lr5e5,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_03_rank8_a16_lr5e5,lmeval/mistral_aya(hruska)/03_rank8_a16_lr5e5/results_2026-10-01T10-25-51.531765.json,0.33105218135158254,0.5167105956773853,0.34550604807839536,0.18380462724935734,0.3424520772406517,0.4715142428785607,36.36378811056779 -04_rank32_a32_lr5e5,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_04_rank32_a32_lr5e5,lmeval/mistral_aya(hruska)/04_rank32_a32_lr5e5/results_2026-10-01T17-22-48.146390.json,0.33532934131736525,0.5200843437005799,0.3449701423977951,0.19023136246786632,0.3367013908863517,0.47301349325337333,36.02282635500946 -05_r16_alpha32_lr5e5,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_05_r16_alpha32_lr5e5,lmeval/mistral_aya(hruska)/05_r16_alpha32_lr5e5/results_2026-10-02T00-19-55.725179.json,0.33361847733105215,0.5190300474433316,0.35714285714285715,0.19665809768637532,0.3555125811399826,0.46701649175412296,37.873456685535025 -06_lr1e4_r16_a16,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_06_lr1e4_r16_a16,lmeval/mistral_aya(hruska)/06_lr1e4_r16_a16/results_2026-10-02T06-59-24.237777.json,0.33105218135158254,0.5207169214549289,0.3549992344204563,0.19023136246786632,0.34733281230911506,0.46551724137931033,37.57962398015711 -07_lr2e5_r16_a16,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_07_lr2e5_r16_a16,lmeval/mistral_aya(hruska)/07_lr2e5_r16_a16/results_2026-10-02T14-27-49.730611.json,0.32763045337895635,0.522509225092251,0.3379268105956209,0.16709511568123395,0.32463255972324384,0.47226386806596704,33.83807391933327 -08_attention_only_r16_a16_lr5e5,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_08_attention_only_r16_a16_lr5e5,lmeval/mistral_aya(hruska)/08_attention_only_r16_a16_lr5e5/results_2026-10-02T19-20-02.720498.json,0.35243798118049613,0.5217712177121772,0.3448170264890522,0.18766066838046272,0.3673805917101876,0.4767616191904048,43.37517960476381 -09_batch2_accum4_r16_a16_lr5e5,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_09_batch2_accum4_r16_a16_lr5e5,lmeval/mistral_aya(hruska)/09_batch2_accum4_r16_a16_lr5e5/results_2026-10-03T01-58-48.046423.json,0.3378956372968349,0.5209277807063785,0.35140101056499773,0.18637532133676094,0.340221777957301,0.46926536731634183,35.3128443275733 -10_r16_a32_lr1e4,100000,/home/schwarc/diplomovka/hruska_compare9_100k_fixed_10_r16_a32_lr1e4,lmeval/mistral_aya(hruska)/10_r16_a32_lr1e4/results_2026-10-03T07-58-30.464881.json,0.32934131736526945,0.515656299420137,0.3448170264890522,0.20565552699228792,0.34795284436431645,0.4617691154422789,37.75612784500817 -aya_300k_lr_0p00010,300000,/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,aya_300k_lr_experiments/benchmarks/lmeval/results/results_2026-10-24T02-28-32.987157.json,0.3473053892215569,0.5163943068002108,0.3102893890675241,0.19665809768637532,0.3506542164316134,0.46101949025487254,38.72920807013749 diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/dataset_access_report.json b/aya_300k_lr_experiments/benchmarks/skbench/configs/dataset_access_report.json deleted file mode 100644 index cc43959..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/dataset_access_report.json +++ /dev/null @@ -1,622 +0,0 @@ -[ - { - "repo": "CohereLabs/Global-MMLU-Lite", - "revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", - "status": "OK", - "error": "", - "tasks": [ - "globalmmlu-lite.prompt3_mqa_single_correct_sk|0", - "globalmmlu-lite.prompt1_mqa_single_correct_sk|0", - "globalmmlu-lite.prompt2_mqa_single_correct_sk|0" - ] - }, - { - "repo": "NaiveNeuron/SlovakCOPA", - "revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", - "status": "OK", - "error": "", - "tasks": [ - "slovakcopa.prompt3_copa_sk|0", - "slovakcopa.prompt1_copa_sk|0", - "slovakcopa.prompt2_copa_sk|0" - ] - }, - { - "repo": "TUKE-KEMT/senti-sk", - "revision": "3c8a92168ff2b692c653de65c97bda573127d15d", - "status": "OK", - "error": "", - "tasks": [ - "senti_sk.prompt2_sentiment_binary_sk|0", - "senti_sk.prompt3_sentiment_binary_sk|0", - "senti_sk.prompt1_sentiment_binary_sk|0", - "senti_sk.prompt3_sentiment_3_class_sk|0", - "senti_sk.prompt1_sentiment_3_class_sk|0", - "senti_sk.prompt2_sentiment_3_class_sk|0" - ] - }, - { - "repo": "TUKE-KEMT/slovak-financial-exam", - "revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", - "status": "OK", - "error": "", - "tasks": [ - "financial_exams.prompt1_completion_sk|0", - "financial_exams.prompt3_completion_sk|0", - "financial_exams.prompt2_completion_sk|0" - ] - }, - { - "repo": "TUKE-KEMT/toxic-sk", - "revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", - "status": "OK", - "error": "", - "tasks": [ - "toxic_sk.prompt2_toxicity_sk|0", - "toxic_sk.prompt3_toxicity_sk|0", - "toxic_sk.prompt1_toxicity_sk|0" - ] - }, - { - "repo": "alexandrainst/m_truthfulqa", - "revision": "f0445d470f1925882b990f5f247fdcf288972f60", - "status": "OK", - "error": "", - "tasks": [ - "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0", - "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0", - "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0" - ] - }, - { - "repo": "alexandrainst/multi-wiki-qa", - "revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", - "status": "OK", - "error": "", - "tasks": [ - "multi_wiki_qa.prompt3_qa_sk|0", - "multi_wiki_qa.prompt2_qa_sk|0", - "multi_wiki_qa.prompt1_qa_sk|0" - ] - }, - { - "repo": "facebook/belebele", - "revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", - "status": "OK", - "error": "", - "tasks": [ - "belebele.prompt3_mqa_single_correct_sk|0", - "belebele.prompt1_mqa_single_correct_sk|0", - "belebele.prompt2_mqa_single_correct_sk|0" - ] - }, - { - "repo": "jon-tow/okapi_arc_challenge", - "revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", - "status": "OK", - "error": "", - "tasks": [ - "arc.prompt3_mqa_single_correct_sk|0", - "arc.prompt1_mqa_single_correct_sk|0", - "arc.prompt2_mqa_single_correct_sk|0" - ] - }, - { - "repo": "lighteval/okapi_mmlu", - "revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "status": "OK", - "error": "", - "tasks": [ - "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0", - "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0", - "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0", - "mmlu.prompt3_mqa_single_correct_sk:anatomy|0", - "mmlu.prompt1_mqa_single_correct_sk:anatomy|0", - "mmlu.prompt2_mqa_single_correct_sk:anatomy|0", - "mmlu.prompt3_mqa_single_correct_sk:astronomy|0", - "mmlu.prompt1_mqa_single_correct_sk:astronomy|0", - "mmlu.prompt2_mqa_single_correct_sk:astronomy|0", - "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0", - "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0", - "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0", - "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0", - "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0", - "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0", - "mmlu.prompt3_mqa_single_correct_sk:college_biology|0", - "mmlu.prompt1_mqa_single_correct_sk:college_biology|0", - "mmlu.prompt2_mqa_single_correct_sk:college_biology|0", - "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0", - "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0", - "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0", - "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0", - "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0", - "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0", - "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0", - "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0", - "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0", - "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0", - "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0", - "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0", - "mmlu.prompt3_mqa_single_correct_sk:college_physics|0", - "mmlu.prompt1_mqa_single_correct_sk:college_physics|0", - "mmlu.prompt2_mqa_single_correct_sk:college_physics|0", - "mmlu.prompt3_mqa_single_correct_sk:computer_security|0", - "mmlu.prompt1_mqa_single_correct_sk:computer_security|0", - "mmlu.prompt2_mqa_single_correct_sk:computer_security|0", - "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0", - "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0", - "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0", - "mmlu.prompt3_mqa_single_correct_sk:econometrics|0", - "mmlu.prompt1_mqa_single_correct_sk:econometrics|0", - "mmlu.prompt2_mqa_single_correct_sk:econometrics|0", - "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0", - "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0", - "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0", - "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0", - "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0", - "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0", - "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0", - "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0", - "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0", - "mmlu.prompt3_mqa_single_correct_sk:global_facts|0", - "mmlu.prompt1_mqa_single_correct_sk:global_facts|0", - "mmlu.prompt2_mqa_single_correct_sk:global_facts|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0", - "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0", - "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0", - "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0", - "mmlu.prompt3_mqa_single_correct_sk:human_aging|0", - "mmlu.prompt1_mqa_single_correct_sk:human_aging|0", - "mmlu.prompt2_mqa_single_correct_sk:human_aging|0", - "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0", - "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0", - "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0", - "mmlu.prompt3_mqa_single_correct_sk:international_law|0", - "mmlu.prompt1_mqa_single_correct_sk:international_law|0", - "mmlu.prompt2_mqa_single_correct_sk:international_law|0", - "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0", - "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0", - "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0", - "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0", - "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0", - "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0", - "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0", - "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0", - "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0", - "mmlu.prompt3_mqa_single_correct_sk:management|0", - "mmlu.prompt1_mqa_single_correct_sk:management|0", - "mmlu.prompt2_mqa_single_correct_sk:management|0", - "mmlu.prompt3_mqa_single_correct_sk:marketing|0", - "mmlu.prompt1_mqa_single_correct_sk:marketing|0", - "mmlu.prompt2_mqa_single_correct_sk:marketing|0", - "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0", - "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0", - "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0", - "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0", - "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0", - "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0", - "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0", - "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0", - "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0", - "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0", - "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0", - "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0", - "mmlu.prompt3_mqa_single_correct_sk:nutrition|0", - "mmlu.prompt1_mqa_single_correct_sk:nutrition|0", - "mmlu.prompt2_mqa_single_correct_sk:nutrition|0", - "mmlu.prompt3_mqa_single_correct_sk:philosophy|0", - "mmlu.prompt1_mqa_single_correct_sk:philosophy|0", - "mmlu.prompt2_mqa_single_correct_sk:philosophy|0", - "mmlu.prompt3_mqa_single_correct_sk:prehistory|0", - "mmlu.prompt1_mqa_single_correct_sk:prehistory|0", - "mmlu.prompt2_mqa_single_correct_sk:prehistory|0", - "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0", - "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0", - "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0", - "mmlu.prompt3_mqa_single_correct_sk:professional_law|0", - "mmlu.prompt1_mqa_single_correct_sk:professional_law|0", - "mmlu.prompt2_mqa_single_correct_sk:professional_law|0", - "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0", - "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0", - "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0", - "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0", - "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0", - "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0", - "mmlu.prompt3_mqa_single_correct_sk:public_relations|0", - "mmlu.prompt1_mqa_single_correct_sk:public_relations|0", - "mmlu.prompt2_mqa_single_correct_sk:public_relations|0", - "mmlu.prompt3_mqa_single_correct_sk:security_studies|0", - "mmlu.prompt1_mqa_single_correct_sk:security_studies|0", - "mmlu.prompt2_mqa_single_correct_sk:security_studies|0", - "mmlu.prompt3_mqa_single_correct_sk:sociology|0", - "mmlu.prompt1_mqa_single_correct_sk:sociology|0", - "mmlu.prompt2_mqa_single_correct_sk:sociology|0", - "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0", - "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0", - "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0", - "mmlu.prompt3_mqa_single_correct_sk:virology|0", - "mmlu.prompt1_mqa_single_correct_sk:virology|0", - "mmlu.prompt2_mqa_single_correct_sk:virology|0", - "mmlu.prompt3_mqa_single_correct_sk:world_religions|0", - "mmlu.prompt1_mqa_single_correct_sk:world_religions|0", - "mmlu.prompt2_mqa_single_correct_sk:world_religions|0" - ] - }, - { - "repo": "mteb/slovak_movie_review_sentiment", - "revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", - "status": "OK", - "error": "", - "tasks": [ - "movie_review.prompt3_sentiment_binary_sk|0", - "movie_review.prompt2_sentiment_binary_sk|0", - "movie_review.prompt1_sentiment_binary_sk|0" - ] - }, - { - "repo": "natalia-nk/NLI-SK-annotated", - "revision": "79914f425b59d8b9fabb6d38c37f6d81f9723f46", - "status": "OK", - "error": "", - "tasks": [ - "nli_sk.prompt1_nli_3_classes_sk|0", - "nli_sk.prompt2_nli_3_classes_sk|0", - "nli_sk.prompt3_nli_3_classes_sk|0" - ] - }, - { - "repo": "slovak-nlp/SKJ1", - "revision": "e615c60b415389b629703f82d3a79810dbde6734", - "status": "OK", - "error": "", - "tasks": [ - "skj1.prompt3_skj1_sk|0", - "skj1.prompt1_skj1_sk|0", - "skj1.prompt2_skj1_sk|0" - ] - }, - { - "repo": "slovak-nlp/chiby-classification", - "revision": "ce8533398057ace6da817758335633067648ca93", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d7-1e7a56917b7b19192d552e13;119d40c0-7b4d-43f3-8cd4-82370a8ee77e)", - "tasks": [ - "chiby.prompt2_chiby_sk|0", - "chiby.prompt1_chiby_sk|0", - "chiby.prompt3_chiby_sk|0" - ] - }, - { - "repo": "slovak-nlp/driving-test-dataset-slovak", - "revision": "37064890fbc00bfca59c035d38f52d265949361e", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d7-3129bdcb539f88e7672e2027;105ec634-cecc-4558-b3c9-e83508219914)", - "tasks": [ - "driving_test.prompt3_mqa_single_correct_sk|0", - "driving_test.prompt1_mqa_single_correct_sk|0", - "driving_test.prompt2_mqa_single_correct_sk|0" - ] - }, - { - "repo": "slovak-nlp/e-testy-sk", - "revision": "1a5e798e0473241097c4a26c511426167522adf5", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d7-4f270b5d6ba17a181d7897ee;9fecd2d5-dc7c-4a4c-8755-61fd6bcda935)", - "tasks": [ - "e_tests.prompt3_mqa_single_correct_sk:biologia|0", - "e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0", - "e_tests.prompt3_mqa_single_correct_sk:informatika|0", - "e_tests.prompt3_mqa_single_correct_sk:geografia|0", - "e_tests.prompt3_mqa_single_correct_sk:matematika|0", - "e_tests.prompt3_mqa_single_correct_sk:chemia|0", - "e_tests.prompt3_mqa_single_correct_sk:dejepis|0", - "e_tests.prompt3_mqa_single_correct_sk:fyzika|0", - "e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0", - "e_tests.prompt1_mqa_single_correct_sk:biologia|0", - "e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0", - "e_tests.prompt1_mqa_single_correct_sk:informatika|0", - "e_tests.prompt1_mqa_single_correct_sk:geografia|0", - "e_tests.prompt1_mqa_single_correct_sk:matematika|0", - "e_tests.prompt1_mqa_single_correct_sk:chemia|0", - "e_tests.prompt1_mqa_single_correct_sk:dejepis|0", - "e_tests.prompt1_mqa_single_correct_sk:fyzika|0", - "e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0", - "e_tests.prompt2_mqa_single_correct_sk:biologia|0", - "e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0", - "e_tests.prompt2_mqa_single_correct_sk:informatika|0", - "e_tests.prompt2_mqa_single_correct_sk:geografia|0", - "e_tests.prompt2_mqa_single_correct_sk:matematika|0", - "e_tests.prompt2_mqa_single_correct_sk:chemia|0", - "e_tests.prompt2_mqa_single_correct_sk:dejepis|0", - "e_tests.prompt2_mqa_single_correct_sk:fyzika|0", - "e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0", - "e_tests.prompt2_mqa_multiple_correct_sk:biologia|0", - "e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0", - "e_tests.prompt2_mqa_multiple_correct_sk:informatika|0", - "e_tests.prompt2_mqa_multiple_correct_sk:geografia|0", - "e_tests.prompt2_mqa_multiple_correct_sk:matematika|0", - "e_tests.prompt2_mqa_multiple_correct_sk:chemia|0", - "e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0", - "e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0", - "e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0", - "e_tests.prompt1_mqa_multiple_correct_sk:biologia|0", - "e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0", - "e_tests.prompt1_mqa_multiple_correct_sk:informatika|0", - "e_tests.prompt1_mqa_multiple_correct_sk:geografia|0", - "e_tests.prompt1_mqa_multiple_correct_sk:matematika|0", - "e_tests.prompt1_mqa_multiple_correct_sk:chemia|0", - "e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0", - "e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0", - "e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0", - "e_tests.prompt3_mqa_multiple_correct_sk:biologia|0", - "e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0", - "e_tests.prompt3_mqa_multiple_correct_sk:informatika|0", - "e_tests.prompt3_mqa_multiple_correct_sk:geografia|0", - "e_tests.prompt3_mqa_multiple_correct_sk:matematika|0", - "e_tests.prompt3_mqa_multiple_correct_sk:chemia|0", - "e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0", - "e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0", - "e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0", - "e_tests.prompt1_qa_sk:biologia|0", - "e_tests.prompt1_qa_sk:obcianska-nauka|0", - "e_tests.prompt1_qa_sk:informatika|0", - "e_tests.prompt1_qa_sk:geografia|0", - "e_tests.prompt1_qa_sk:chemia|0", - "e_tests.prompt1_qa_sk:dejepis|0", - "e_tests.prompt1_qa_sk:fyzika|0", - "e_tests.prompt1_qa_sk:slovensky-jazyk|0", - "e_tests.prompt3_qa_sk:biologia|0", - "e_tests.prompt3_qa_sk:obcianska-nauka|0", - "e_tests.prompt3_qa_sk:informatika|0", - "e_tests.prompt3_qa_sk:geografia|0", - "e_tests.prompt3_qa_sk:chemia|0", - "e_tests.prompt3_qa_sk:dejepis|0", - "e_tests.prompt3_qa_sk:fyzika|0", - "e_tests.prompt3_qa_sk:slovensky-jazyk|0", - "e_tests.prompt2_qa_sk:biologia|0", - "e_tests.prompt2_qa_sk:obcianska-nauka|0", - "e_tests.prompt2_qa_sk:informatika|0", - "e_tests.prompt2_qa_sk:geografia|0", - "e_tests.prompt2_qa_sk:chemia|0", - "e_tests.prompt2_qa_sk:dejepis|0", - "e_tests.prompt2_qa_sk:fyzika|0", - "e_tests.prompt2_qa_sk:slovensky-jazyk|0" - ] - }, - { - "repo": "slovak-nlp/ifeval-sk", - "revision": "2114c70fdcd22e8e3e5ea11f6b77067740959bf4", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d7-5666942d3a4121b676a928ff;cbd91d44-6ee2-44b0-b74b-3aca15012a31)", - "tasks": [ - "ifeval_sk.ifeval|0" - ] - }, - { - "repo": "slovak-nlp/oneruler-sk-processed", - "revision": "c3a160bc7ce2dada86803f26ee699ca75ee26e41", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d7-342e828a639d5d19427fae4f;91605400-c335-4c30-8e40-78bdf0e40b3c)", - "tasks": [ - "oneruler_sk.oneruler_slovak:cwe_4096|0", - "oneruler_sk.oneruler_slovak:cwe_8192|0", - "oneruler_sk.oneruler_slovak:cwe_16384|0", - "oneruler_sk.oneruler_slovak:niah_multikey_4096|0", - "oneruler_sk.oneruler_slovak:niah_multikey_8192|0", - "oneruler_sk.oneruler_slovak:niah_multikey_16384|0", - "oneruler_sk.oneruler_slovak:niah_multiquery_4096|0", - "oneruler_sk.oneruler_slovak:niah_multiquery_8192|0", - "oneruler_sk.oneruler_slovak:niah_multiquery_16384|0", - "oneruler_sk.oneruler_slovak:niah_multivalue_4096|0", - "oneruler_sk.oneruler_slovak:niah_multivalue_8192|0", - "oneruler_sk.oneruler_slovak:niah_multivalue_16384|0", - "oneruler_sk.oneruler_slovak:niah_none_4096|0", - "oneruler_sk.oneruler_slovak:niah_none_8192|0", - "oneruler_sk.oneruler_slovak:niah_none_16384|0", - "oneruler_sk.oneruler_slovak:niah_single_4096|0", - "oneruler_sk.oneruler_slovak:niah_single_8192|0", - "oneruler_sk.oneruler_slovak:niah_single_16384|0" - ] - }, - { - "repo": "slovak-nlp/scio-testy", - "revision": "78c67eeae5d48b92590945a1c589b7730bfce8df", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d7-203c6cfb4d094ab1703aa786;3f87f5ab-83e3-4b52-b012-e6e26eed3d43)", - "tasks": [ - "scio.prompt2_mqa_single_correct_sk:aj|0", - "scio.prompt2_mqa_single_correct_sk:mat|0", - "scio.prompt2_mqa_single_correct_sk:bio|0", - "scio.prompt2_mqa_single_correct_sk:che|0", - "scio.prompt2_mqa_single_correct_sk:vsp|0", - "scio.prompt3_mqa_single_correct_sk:aj|0", - "scio.prompt3_mqa_single_correct_sk:mat|0", - "scio.prompt3_mqa_single_correct_sk:bio|0", - "scio.prompt3_mqa_single_correct_sk:che|0", - "scio.prompt3_mqa_single_correct_sk:vsp|0", - "scio.prompt1_mqa_single_correct_sk:aj|0", - "scio.prompt1_mqa_single_correct_sk:mat|0", - "scio.prompt1_mqa_single_correct_sk:bio|0", - "scio.prompt1_mqa_single_correct_sk:che|0", - "scio.prompt1_mqa_single_correct_sk:vsp|0" - ] - }, - { - "repo": "slovak-nlp/sklep", - "revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "status": "OK", - "error": "", - "tasks": [ - "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0", - "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0", - "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0", - "sklep.prompt1_hate_speech_sk:hate-speech|0", - "sklep.prompt3_hate_speech_sk:hate-speech|0", - "sklep.prompt2_hate_speech_sk:hate-speech|0", - "sklep.prompt1_nli_3_classes_sk:nli|0", - "sklep.prompt2_nli_3_classes_sk:nli|0", - "sklep.prompt3_nli_3_classes_sk:nli|0", - "sklep.prompt1_nli_binary_sk:rte|0", - "sklep.prompt3_nli_binary_sk:rte|0", - "sklep.prompt2_nli_binary_sk:rte|0", - "sklep.prompt3_qa_sk:question-answering|0", - "sklep.prompt2_qa_sk:question-answering|0", - "sklep.prompt1_qa_sk:question-answering|0" - ] - }, - { - "repo": "slovak-nlp/slohry", - "revision": "3b47acd8a549c65ec6ab34aacd538f3ebcbfa32f", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d7-18026919100903a2056ca192;a073f7f3-9b00-49ca-982c-c03741ca2364)", - "tasks": [ - "slohry.prompt2_declension_sk:declension_high|0", - "slohry.prompt2_declension_sk:declension_mid|0", - "slohry.prompt2_declension_sk:declension_low|0", - "slohry.prompt1_declension_sk:declension_high|0", - "slohry.prompt1_declension_sk:declension_mid|0", - "slohry.prompt1_declension_sk:declension_low|0", - "slohry.prompt3_declension_sk:declension_high|0", - "slohry.prompt3_declension_sk:declension_mid|0", - "slohry.prompt3_declension_sk:declension_low|0", - "slohry.prompt3_gradation_sk:gradation_high|0", - "slohry.prompt3_gradation_sk:gradation_mid|0", - "slohry.prompt3_gradation_sk:gradation_low|0", - "slohry.prompt1_gradation_sk:gradation_high|0", - "slohry.prompt1_gradation_sk:gradation_mid|0", - "slohry.prompt1_gradation_sk:gradation_low|0", - "slohry.prompt2_gradation_sk:gradation_high|0", - "slohry.prompt2_gradation_sk:gradation_mid|0", - "slohry.prompt2_gradation_sk:gradation_low|0", - "slohry.prompt1_identity_form_sk:identity_form_high|0", - "slohry.prompt1_identity_form_sk:identity_form_mid|0", - "slohry.prompt1_identity_form_sk:identity_form_low|0", - "slohry.prompt2_identity_form_sk:identity_form_high|0", - "slohry.prompt2_identity_form_sk:identity_form_mid|0", - "slohry.prompt2_identity_form_sk:identity_form_low|0", - "slohry.prompt3_identity_form_sk:identity_form_high|0", - "slohry.prompt3_identity_form_sk:identity_form_mid|0", - "slohry.prompt3_identity_form_sk:identity_form_low|0", - "slohry.prompt2_inflection_sk:inflection_high|0", - "slohry.prompt2_inflection_sk:inflection_mid|0", - "slohry.prompt2_inflection_sk:inflection_low|0", - "slohry.prompt1_inflection_sk:inflection_high|0", - "slohry.prompt1_inflection_sk:inflection_mid|0", - "slohry.prompt1_inflection_sk:inflection_low|0", - "slohry.prompt3_inflection_sk:inflection_high|0", - "slohry.prompt3_inflection_sk:inflection_mid|0", - "slohry.prompt3_inflection_sk:inflection_low|0" - ] - }, - { - "repo": "slovak-nlp/slovak-school-exams-test-preprocessed", - "revision": "1d51f8205ee2341d11f05273d9c6d4ba3532a1df", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d8-7d40d0951ba637dd705fbdd4;c5ed7cb3-8e89-455e-af3d-699ae41c323d)", - "tasks": [ - "slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0", - "slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0", - "slovak_tests.prompt3_mqa_single_correct_sk:maturita|0", - "slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0", - "slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0", - "slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0", - "slovak_tests.prompt1_mqa_single_correct_sk:maturita|0", - "slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0", - "slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0", - "slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0", - "slovak_tests.prompt2_mqa_single_correct_sk:maturita|0", - "slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0", - "slovak_tests.prompt1_qa_sk:monitor9|0", - "slovak_tests.prompt1_qa_sk:maturita|0", - "slovak_tests.prompt1_qa_sk:prijmacie_skusky|0", - "slovak_tests.prompt3_qa_sk:monitor9|0", - "slovak_tests.prompt3_qa_sk:maturita|0", - "slovak_tests.prompt3_qa_sk:prijmacie_skusky|0", - "slovak_tests.prompt2_qa_sk:monitor9|0", - "slovak_tests.prompt2_qa_sk:maturita|0", - "slovak_tests.prompt2_qa_sk:prijmacie_skusky|0" - ] - }, - { - "repo": "slovak-nlp/slovak_trivia", - "revision": "2f27ff5a3c266a379c164d3cb43fc9cf8e45b37a", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d8-60bbc6ed1f90512f2a40bf37;5a9d9ee9-412d-4cdd-a8a3-66accb5e4d3e)", - "tasks": [ - "slovak_trivia.prompt2_mqa_single_correct_sk|0", - "slovak_trivia.prompt3_mqa_single_correct_sk|0", - "slovak_trivia.prompt1_mqa_single_correct_sk|0" - ] - }, - { - "repo": "slovak-nlp/slovaksum-downsampled", - "revision": "2e4cdabe972ac040114701ea9e885e0a648a33fb", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d8-3266710308ee05503220c4de;bd8592d6-e022-40ef-a2dd-c9bd490e957f)", - "tasks": [ - "slovaksum.prompt2_summarization_sk|0", - "slovaksum.prompt3_summarization_sk|0", - "slovaksum.prompt1_summarization_sk|0" - ] - }, - { - "repo": "slovak-nlp/smesum-downsampled", - "revision": "cb427bdc96e444637508fb982915f706c19d81d4", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d8-4c1e6f5602eda70b00239325;126975ea-e381-456f-8fa6-0ffff90f3938)", - "tasks": [ - "smesum.prompt2_summarization_sk|0", - "smesum.prompt3_summarization_sk|0", - "smesum.prompt1_summarization_sk|0" - ] - }, - { - "repo": "slovak-nlp/word-analogies-sk", - "revision": "d6564d49c07b29ea93028f55a57c25eb231762f9", - "status": "NEDOSTUPNÉ", - "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6ada58d8-18fc3a1223dc59e9238fa860;c8c94f13-fd96-4630-9510-c1ab0db7a366)", - "tasks": [ - "analogies_sk.prompt1_analogy_sk|0", - "analogies_sk.prompt2_analogy_sk|0", - "analogies_sk.prompt3_analogy_sk|0" - ] - } -] \ No newline at end of file diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/export_lr_0p00010.yaml b/aya_300k_lr_experiments/benchmarks/skbench/configs/export_lr_0p00010.yaml deleted file mode 100644 index fcbcfa7..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/export_lr_0p00010.yaml +++ /dev/null @@ -1,12 +0,0 @@ -model_name_or_path: slovak-nlp/mistral-sk-7b -model_revision: 089497ae0d72018e9591895c54f774bf0b4a83a4 -adapter_name_or_path: /home/schwarc/diplomovka/aya_300k_lr_validation/lr_0p00010 - -template: alpaca -finetuning_type: lora -infer_dtype: bfloat16 - -export_dir: /home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010 -export_size: 5 -export_device: cpu -export_legacy_format: false diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/model_config.json b/aya_300k_lr_experiments/benchmarks/skbench/configs/model_config.json deleted file mode 100644 index ebbb6ba..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/model_config.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "lighteval_sha": "?", - "num_fewshot_seeds": 1, - "max_samples": null, - "job_id": "0", - "start_time": 796668.90423246, - "end_time": 807588.555648873, - "total_evaluation_time_secondes": "10919.65141641302", - "model_config": { - "model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010", - "generation_parameters": { - "num_blocks": null, - "block_size": null, - "early_stopping": null, - "repetition_penalty": null, - "frequency_penalty": null, - "length_penalty": null, - "presence_penalty": null, - "max_new_tokens": null, - "min_new_tokens": null, - "seed": null, - "stop_tokens": null, - "temperature": 0, - "top_k": null, - "min_p": null, - "top_p": null, - "truncate_prompt": null, - "cache_implementation": null, - "response_format": null, - "skip_special_tokens": true - }, - "system_prompt": null, - "cache_dir": "~/.cache/huggingface/lighteval", - "tokenizer": null, - "revision": "main", - "dtype": "bfloat16", - "tensor_parallel_size": 1, - "data_parallel_size": 1, - "pipeline_parallel_size": 1, - "gpu_memory_utilization": 0.85, - "enable_prefix_caching": null, - "max_model_length": 32768, - "quantization": null, - "load_format": null, - "swap_space": 4, - "seed": 1234, - "trust_remote_code": false, - "add_special_tokens": true, - "multichoice_continuations_start_space": true, - "pairwise_tokenization": false, - "max_num_seqs": 128, - "max_num_batched_tokens": 2048, - "subfolder": null, - "is_async": false, - "override_chat_template": null, - "enforce_eager": true - }, - "model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010" -} diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/omitted_tasks.txt b/aya_300k_lr_experiments/benchmarks/skbench/configs/omitted_tasks.txt deleted file mode 100644 index b7b2fdb..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/omitted_tasks.txt +++ /dev/null @@ -1,190 +0,0 @@ -chiby.prompt2_chiby_sk|0 -chiby.prompt1_chiby_sk|0 -chiby.prompt3_chiby_sk|0 -ifeval_sk.ifeval|0 -oneruler_sk.oneruler_slovak:cwe_4096|0 -oneruler_sk.oneruler_slovak:cwe_8192|0 -oneruler_sk.oneruler_slovak:cwe_16384|0 -oneruler_sk.oneruler_slovak:niah_multikey_4096|0 -oneruler_sk.oneruler_slovak:niah_multikey_8192|0 -oneruler_sk.oneruler_slovak:niah_multikey_16384|0 -oneruler_sk.oneruler_slovak:niah_multiquery_4096|0 -oneruler_sk.oneruler_slovak:niah_multiquery_8192|0 -oneruler_sk.oneruler_slovak:niah_multiquery_16384|0 -oneruler_sk.oneruler_slovak:niah_multivalue_4096|0 -oneruler_sk.oneruler_slovak:niah_multivalue_8192|0 -oneruler_sk.oneruler_slovak:niah_multivalue_16384|0 -oneruler_sk.oneruler_slovak:niah_none_4096|0 -oneruler_sk.oneruler_slovak:niah_none_8192|0 -oneruler_sk.oneruler_slovak:niah_none_16384|0 -oneruler_sk.oneruler_slovak:niah_single_4096|0 -oneruler_sk.oneruler_slovak:niah_single_8192|0 -oneruler_sk.oneruler_slovak:niah_single_16384|0 -slovaksum.prompt2_summarization_sk|0 -slovaksum.prompt3_summarization_sk|0 -slovaksum.prompt1_summarization_sk|0 -smesum.prompt2_summarization_sk|0 -smesum.prompt3_summarization_sk|0 -smesum.prompt1_summarization_sk|0 -slovak_trivia.prompt2_mqa_single_correct_sk|0 -slovak_trivia.prompt3_mqa_single_correct_sk|0 -slovak_trivia.prompt1_mqa_single_correct_sk|0 -analogies_sk.prompt1_analogy_sk|0 -analogies_sk.prompt2_analogy_sk|0 -analogies_sk.prompt3_analogy_sk|0 -driving_test.prompt3_mqa_single_correct_sk|0 -driving_test.prompt1_mqa_single_correct_sk|0 -driving_test.prompt2_mqa_single_correct_sk|0 -e_tests.prompt3_mqa_single_correct_sk:biologia|0 -e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt3_mqa_single_correct_sk:informatika|0 -e_tests.prompt3_mqa_single_correct_sk:geografia|0 -e_tests.prompt3_mqa_single_correct_sk:matematika|0 -e_tests.prompt3_mqa_single_correct_sk:chemia|0 -e_tests.prompt3_mqa_single_correct_sk:dejepis|0 -e_tests.prompt3_mqa_single_correct_sk:fyzika|0 -e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt1_mqa_single_correct_sk:biologia|0 -e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt1_mqa_single_correct_sk:informatika|0 -e_tests.prompt1_mqa_single_correct_sk:geografia|0 -e_tests.prompt1_mqa_single_correct_sk:matematika|0 -e_tests.prompt1_mqa_single_correct_sk:chemia|0 -e_tests.prompt1_mqa_single_correct_sk:dejepis|0 -e_tests.prompt1_mqa_single_correct_sk:fyzika|0 -e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt2_mqa_single_correct_sk:biologia|0 -e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt2_mqa_single_correct_sk:informatika|0 -e_tests.prompt2_mqa_single_correct_sk:geografia|0 -e_tests.prompt2_mqa_single_correct_sk:matematika|0 -e_tests.prompt2_mqa_single_correct_sk:chemia|0 -e_tests.prompt2_mqa_single_correct_sk:dejepis|0 -e_tests.prompt2_mqa_single_correct_sk:fyzika|0 -e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt2_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt2_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt2_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt2_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt2_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0 -e_tests.prompt1_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt1_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt1_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt1_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt1_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0 -e_tests.prompt3_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt3_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt3_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt3_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt3_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0 -slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt3_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0 -slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt1_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0 -slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt2_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0 -e_tests.prompt1_qa_sk:biologia|0 -e_tests.prompt1_qa_sk:obcianska-nauka|0 -e_tests.prompt1_qa_sk:informatika|0 -e_tests.prompt1_qa_sk:geografia|0 -e_tests.prompt1_qa_sk:chemia|0 -e_tests.prompt1_qa_sk:dejepis|0 -e_tests.prompt1_qa_sk:fyzika|0 -e_tests.prompt1_qa_sk:slovensky-jazyk|0 -e_tests.prompt3_qa_sk:biologia|0 -e_tests.prompt3_qa_sk:obcianska-nauka|0 -e_tests.prompt3_qa_sk:informatika|0 -e_tests.prompt3_qa_sk:geografia|0 -e_tests.prompt3_qa_sk:chemia|0 -e_tests.prompt3_qa_sk:dejepis|0 -e_tests.prompt3_qa_sk:fyzika|0 -e_tests.prompt3_qa_sk:slovensky-jazyk|0 -e_tests.prompt2_qa_sk:biologia|0 -e_tests.prompt2_qa_sk:obcianska-nauka|0 -e_tests.prompt2_qa_sk:informatika|0 -e_tests.prompt2_qa_sk:geografia|0 -e_tests.prompt2_qa_sk:chemia|0 -e_tests.prompt2_qa_sk:dejepis|0 -e_tests.prompt2_qa_sk:fyzika|0 -e_tests.prompt2_qa_sk:slovensky-jazyk|0 -slovak_tests.prompt1_qa_sk:monitor9|0 -slovak_tests.prompt1_qa_sk:maturita|0 -slovak_tests.prompt1_qa_sk:prijmacie_skusky|0 -slovak_tests.prompt3_qa_sk:monitor9|0 -slovak_tests.prompt3_qa_sk:maturita|0 -slovak_tests.prompt3_qa_sk:prijmacie_skusky|0 -slovak_tests.prompt2_qa_sk:monitor9|0 -slovak_tests.prompt2_qa_sk:maturita|0 -slovak_tests.prompt2_qa_sk:prijmacie_skusky|0 -slohry.prompt2_declension_sk:declension_high|0 -slohry.prompt2_declension_sk:declension_mid|0 -slohry.prompt2_declension_sk:declension_low|0 -slohry.prompt1_declension_sk:declension_high|0 -slohry.prompt1_declension_sk:declension_mid|0 -slohry.prompt1_declension_sk:declension_low|0 -slohry.prompt3_declension_sk:declension_high|0 -slohry.prompt3_declension_sk:declension_mid|0 -slohry.prompt3_declension_sk:declension_low|0 -slohry.prompt3_gradation_sk:gradation_high|0 -slohry.prompt3_gradation_sk:gradation_mid|0 -slohry.prompt3_gradation_sk:gradation_low|0 -slohry.prompt1_gradation_sk:gradation_high|0 -slohry.prompt1_gradation_sk:gradation_mid|0 -slohry.prompt1_gradation_sk:gradation_low|0 -slohry.prompt2_gradation_sk:gradation_high|0 -slohry.prompt2_gradation_sk:gradation_mid|0 -slohry.prompt2_gradation_sk:gradation_low|0 -slohry.prompt1_identity_form_sk:identity_form_high|0 -slohry.prompt1_identity_form_sk:identity_form_mid|0 -slohry.prompt1_identity_form_sk:identity_form_low|0 -slohry.prompt2_identity_form_sk:identity_form_high|0 -slohry.prompt2_identity_form_sk:identity_form_mid|0 -slohry.prompt2_identity_form_sk:identity_form_low|0 -slohry.prompt3_identity_form_sk:identity_form_high|0 -slohry.prompt3_identity_form_sk:identity_form_mid|0 -slohry.prompt3_identity_form_sk:identity_form_low|0 -slohry.prompt2_inflection_sk:inflection_high|0 -slohry.prompt2_inflection_sk:inflection_mid|0 -slohry.prompt2_inflection_sk:inflection_low|0 -slohry.prompt1_inflection_sk:inflection_high|0 -slohry.prompt1_inflection_sk:inflection_mid|0 -slohry.prompt1_inflection_sk:inflection_low|0 -slohry.prompt3_inflection_sk:inflection_high|0 -slohry.prompt3_inflection_sk:inflection_mid|0 -slohry.prompt3_inflection_sk:inflection_low|0 -scio.prompt2_mqa_single_correct_sk:aj|0 -scio.prompt2_mqa_single_correct_sk:mat|0 -scio.prompt2_mqa_single_correct_sk:bio|0 -scio.prompt2_mqa_single_correct_sk:che|0 -scio.prompt2_mqa_single_correct_sk:vsp|0 -scio.prompt3_mqa_single_correct_sk:aj|0 -scio.prompt3_mqa_single_correct_sk:mat|0 -scio.prompt3_mqa_single_correct_sk:bio|0 -scio.prompt3_mqa_single_correct_sk:che|0 -scio.prompt3_mqa_single_correct_sk:vsp|0 -scio.prompt1_mqa_single_correct_sk:aj|0 -scio.prompt1_mqa_single_correct_sk:mat|0 -scio.prompt1_mqa_single_correct_sk:bio|0 -scio.prompt1_mqa_single_correct_sk:che|0 -scio.prompt1_mqa_single_correct_sk:vsp|0 -nli_sk.prompt1_nli_3_classes_sk|0 -nli_sk.prompt2_nli_3_classes_sk|0 -nli_sk.prompt3_nli_3_classes_sk|0 diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/task_configs.json b/aya_300k_lr_experiments/benchmarks/skbench/configs/task_configs.json deleted file mode 100644 index a7c580e..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/task_configs.json +++ /dev/null @@ -1,13475 +0,0 @@ -{ - "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:miscellaneous", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "financial_exams.prompt2_completion_sk|0": { - "name": "financial_exams.prompt2_completion_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/slovak-financial-exam", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", - "hf_filter": null, - "hf_avail_splits": [ - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:machine_learning", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "belebele.prompt3_mqa_single_correct_sk|0": { - "name": "belebele.prompt3_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "facebook/belebele", - "hf_subset": "slk_Latn", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", - "hf_filter": null, - "hf_avail_splits": [ - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:professional_law", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:college_computer_science", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:professional_accounting", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:professional_law", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt1_qa_sk:question-answering|0": { - "name": "sklep.prompt1_qa_sk:question-answering", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "question-answering", - "metrics": [ - { - "metric_name": "exact_match_slk_prefix", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "f1_slk", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 126, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:college_chemistry", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:college_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:professional_medicine", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "slovakcopa.prompt3_copa_sk|0": { - "name": "slovakcopa.prompt3_copa_sk", - "prompt_function": "prompt_fn", - "hf_repo": "NaiveNeuron/SlovakCOPA", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "validation", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "movie_review.prompt2_sentiment_binary_sk|0": { - "name": "movie_review.prompt2_sentiment_binary_sk", - "prompt_function": "prompt_fn", - "hf_repo": "mteb/slovak_movie_review_sentiment", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt2_qa_sk:question-answering|0": { - "name": "sklep.prompt2_qa_sk:question-answering", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "question-answering", - "metrics": [ - { - "metric_name": "exact_match_slk_prefix", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "f1_slk", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 126, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:college_biology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:nutrition", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_biology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "arc.prompt3_mqa_single_correct_sk|0": { - "name": "arc.prompt3_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "jon-tow/okapi_arc_challenge", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": [ - "solve" - ], - "scorer": "score", - "sample_fields": "record_to_sample", - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:econometrics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:prehistory", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "toxic_sk.prompt3_toxicity_sk|0": { - "name": "toxic_sk.prompt3_toxicity_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/toxic-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "belebele.prompt1_mqa_single_correct_sk|0": { - "name": "belebele.prompt1_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "facebook/belebele", - "hf_subset": "slk_Latn", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", - "hf_filter": null, - "hf_avail_splits": [ - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "arc.prompt2_mqa_single_correct_sk|0": { - "name": "arc.prompt2_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "jon-tow/okapi_arc_challenge", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": [ - "solve" - ], - "scorer": "score", - "sample_fields": "record_to_sample", - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "multi_wiki_qa.prompt2_qa_sk|0": { - "name": "multi_wiki_qa.prompt2_qa_sk", - "prompt_function": "prompt_fn", - "hf_repo": "alexandrainst/multi-wiki-qa", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "exact_match_slk_prefix", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "f1_slk", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", - "hf_filter": "", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "train" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 126, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:management|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:management", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:college_chemistry", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt3_nli_3_classes_sk:nli|0": { - "name": "sklep.prompt3_nli_3_classes_sk:nli", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "nli", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:college_medicine", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:business_ethics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:college_computer_science", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:international_law", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:international_law", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { - "name": "truthfulqa.prompt3_mqa_single_correct_sk:mc1", - "prompt_function": "prompt_fn", - "hf_repo": "alexandrainst/m_truthfulqa", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "val" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:prehistory", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:nutrition", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:college_computer_science", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:college_medicine", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:college_medicine", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt2_nli_binary_sk:rte|0": { - "name": "sklep.prompt2_nli_binary_sk:rte", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "rte", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt2_hate_speech_sk:hate-speech|0": { - "name": "sklep.prompt2_hate_speech_sk:hate-speech", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "hate-speech", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:human_sexuality", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:marketing", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:virology|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:virology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:medical_genetics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:world_religions", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "senti_sk.prompt3_sentiment_binary_sk|0": { - "name": "senti_sk.prompt3_sentiment_binary_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/senti-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", - "hf_filter": "", - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt1_nli_3_classes_sk:nli|0": { - "name": "sklep.prompt1_nli_3_classes_sk:nli", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "nli", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:business_ethics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:public_relations", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "senti_sk.prompt2_sentiment_binary_sk|0": { - "name": "senti_sk.prompt2_sentiment_binary_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/senti-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", - "hf_filter": "", - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:medical_genetics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "belebele.prompt2_mqa_single_correct_sk|0": { - "name": "belebele.prompt2_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "facebook/belebele", - "hf_subset": "slk_Latn", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", - "hf_filter": null, - "hf_avail_splits": [ - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:virology|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:virology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt2_nli_3_classes_sk:nli|0": { - "name": "sklep.prompt2_nli_3_classes_sk:nli", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "nli", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { - "name": "sklep.prompt3_sentiment_binary_sk:sentiment-analysis", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "sentiment-analysis", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:moral_disputes", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:security_studies", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "skj1.prompt1_skj1_sk|0": { - "name": "skj1.prompt1_skj1_sk", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/SKJ1", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", - "hf_filter": null, - "hf_avail_splits": [ - "train" - ], - "evaluation_splits": [ - "train" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:college_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt3_hate_speech_sk:hate-speech|0": { - "name": "sklep.prompt3_hate_speech_sk:hate-speech", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "hate-speech", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:virology|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:virology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:security_studies", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "financial_exams.prompt3_completion_sk|0": { - "name": "financial_exams.prompt3_completion_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/slovak-financial-exam", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", - "hf_filter": null, - "hf_avail_splits": [ - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:jurisprudence", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "senti_sk.prompt3_sentiment_3_class_sk|0": { - "name": "senti_sk.prompt3_sentiment_3_class_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/senti-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt3_qa_sk:question-answering|0": { - "name": "sklep.prompt3_qa_sk:question-answering", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "question-answering", - "metrics": [ - { - "metric_name": "exact_match_slk_prefix", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "f1_slk", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 126, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:professional_psychology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:global_facts", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:computer_security", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:computer_security", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:college_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { - "name": "globalmmlu-lite.prompt2_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "CohereLabs/Global-MMLU-Lite", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": "1" - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:anatomy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:security_studies", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "movie_review.prompt3_sentiment_binary_sk|0": { - "name": "movie_review.prompt3_sentiment_binary_sk", - "prompt_function": "prompt_fn", - "hf_repo": "mteb/slovak_movie_review_sentiment", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:college_chemistry", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:astronomy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:medical_genetics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:philosophy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:human_sexuality", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:world_religions", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "toxic_sk.prompt1_toxicity_sk|0": { - "name": "toxic_sk.prompt1_toxicity_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/toxic-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "senti_sk.prompt1_sentiment_3_class_sk|0": { - "name": "senti_sk.prompt1_sentiment_3_class_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/senti-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:college_biology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:public_relations", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:formal_logic", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_geography", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:prehistory", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:econometrics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "arc.prompt1_mqa_single_correct_sk|0": { - "name": "arc.prompt1_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "jon-tow/okapi_arc_challenge", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": [ - "solve" - ], - "scorer": "score", - "sample_fields": "record_to_sample", - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:business_ethics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:jurisprudence", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { - "name": "truthfulqa.prompt2_mqa_single_correct_sk:mc1", - "prompt_function": "prompt_fn", - "hf_repo": "alexandrainst/m_truthfulqa", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "val" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:miscellaneous", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:astronomy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "senti_sk.prompt2_sentiment_3_class_sk|0": { - "name": "senti_sk.prompt2_sentiment_3_class_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/senti-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:formal_logic", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { - "name": "sklep.prompt2_sentiment_binary_sk:sentiment-analysis", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "sentiment-analysis", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { - "name": "sklep.prompt1_sentiment_binary_sk:sentiment-analysis", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "sentiment-analysis", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "skj1.prompt2_skj1_sk|0": { - "name": "skj1.prompt2_skj1_sk", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/SKJ1", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", - "hf_filter": null, - "hf_avail_splits": [ - "train" - ], - "evaluation_splits": [ - "train" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:international_law", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt1_nli_binary_sk:rte|0": { - "name": "sklep.prompt1_nli_binary_sk:rte", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "rte", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:human_aging", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:management|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:management", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:professional_accounting", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { - "name": "truthfulqa.prompt1_mqa_single_correct_sk:mc1", - "prompt_function": "prompt_fn", - "hf_repo": "alexandrainst/m_truthfulqa", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "val" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "senti_sk.prompt1_sentiment_binary_sk|0": { - "name": "senti_sk.prompt1_sentiment_binary_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/senti-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", - "hf_filter": "", - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:astronomy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:human_aging", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:formal_logic", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:sociology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:econometrics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:college_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:professional_psychology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:college_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { - "name": "globalmmlu-lite.prompt3_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "CohereLabs/Global-MMLU-Lite", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": "1" - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:human_sexuality", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:computer_security", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "toxic_sk.prompt2_toxicity_sk|0": { - "name": "toxic_sk.prompt2_toxicity_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/toxic-sk", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "multi_wiki_qa.prompt3_qa_sk|0": { - "name": "multi_wiki_qa.prompt3_qa_sk", - "prompt_function": "prompt_fn", - "hf_repo": "alexandrainst/multi-wiki-qa", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "exact_match_slk_prefix", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "f1_slk", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", - "hf_filter": "", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "train" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 126, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "financial_exams.prompt1_completion_sk|0": { - "name": "financial_exams.prompt1_completion_sk", - "prompt_function": "prompt_fn", - "hf_repo": "TUKE-KEMT/slovak-financial-exam", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", - "hf_filter": null, - "hf_avail_splits": [ - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:philosophy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:moral_disputes", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:marketing", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { - "name": "globalmmlu-lite.prompt1_mqa_single_correct_sk", - "prompt_function": "prompt_fn", - "hf_repo": "CohereLabs/Global-MMLU-Lite", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": "1" - }, - "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:professional_law", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:machine_learning", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:management|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:management", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:sociology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "skj1.prompt3_skj1_sk|0": { - "name": "skj1.prompt3_skj1_sk", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/SKJ1", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", - "hf_filter": null, - "hf_avail_splits": [ - "train" - ], - "evaluation_splits": [ - "train" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "movie_review.prompt1_sentiment_binary_sk|0": { - "name": "movie_review.prompt1_sentiment_binary_sk", - "prompt_function": "prompt_fn", - "hf_repo": "mteb/slovak_movie_review_sentiment", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:anatomy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:philosophy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:human_aging", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:professional_medicine", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:professional_psychology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt1_hate_speech_sk:hate-speech|0": { - "name": "sklep.prompt1_hate_speech_sk:hate-speech", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "hate-speech", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:miscellaneous", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:world_religions", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_biology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:college_biology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:global_facts", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:moral_disputes", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_geography", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:college_physics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:professional_medicine", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "sklep.prompt3_nli_binary_sk:rte|0": { - "name": "sklep.prompt3_nli_binary_sk:rte", - "prompt_function": "prompt_fn", - "hf_repo": "slovak-nlp/sklep", - "hf_subset": "rte", - "metrics": [ - { - "metric_name": "em_norm_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative_norm", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "train", - "few_shots_select": "random_sampling_from_train", - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "slovakcopa.prompt2_copa_sk|0": { - "name": "slovakcopa.prompt2_copa_sk", - "prompt_function": "prompt_fn", - "hf_repo": "NaiveNeuron/SlovakCOPA", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "validation", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:jurisprudence", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "multi_wiki_qa.prompt1_qa_sk|0": { - "name": "multi_wiki_qa.prompt1_qa_sk", - "prompt_function": "prompt_fn", - "hf_repo": "alexandrainst/multi-wiki-qa", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "exact_match_slk_prefix", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "f1_slk", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", - "hf_filter": "", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "train" - ], - "few_shots_split": null, - "few_shots_select": null, - "generation_size": 126, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:global_facts", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:sociology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_biology", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { - "name": "mmlu.prompt1_mqa_single_correct_sk:public_relations", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:anatomy", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:marketing", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:professional_accounting", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_geography", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "slovakcopa.prompt1_copa_sk|0": { - "name": "slovakcopa.prompt1_copa_sk", - "prompt_function": "prompt_fn", - "hf_repo": "NaiveNeuron/SlovakCOPA", - "hf_subset": "default", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", - "hf_filter": null, - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "validation", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { - "name": "mmlu.prompt2_mqa_single_correct_sk:nutrition", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - }, - "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { - "name": "mmlu.prompt3_mqa_single_correct_sk:machine_learning", - "prompt_function": "prompt_fn", - "hf_repo": "lighteval/okapi_mmlu", - "hf_subset": "sk", - "metrics": [ - { - "metric_name": "em_relaxed", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", - "corpus_level_fn": "mean", - "batched_compute": false - }, - { - "metric_name": "em_conservative", - "higher_is_better": true, - "category": "GENERATIVE", - "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", - "corpus_level_fn": "mean", - "batched_compute": false - } - ], - "hf_data_files": null, - "solver": null, - "scorer": null, - "sample_fields": null, - "sample_to_fewshot": null, - "filter": null, - "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", - "hf_filter": "mmlu_filter", - "hf_avail_splits": [ - "train", - "validation", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": "dev", - "few_shots_select": null, - "generation_size": 32, - "generation_grammar": null, - "stop_sequence": [ - "\n" - ], - "num_samples": null, - "original_num_docs": -1, - "effective_num_docs": -1, - "must_remove_duplicate_docs": false, - "num_fewshots": 0, - "version": 0 - } -} diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks.txt b/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks.txt deleted file mode 100644 index 45b85f6..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks.txt +++ /dev/null @@ -1,222 +0,0 @@ -arc.prompt3_mqa_single_correct_sk|0 -arc.prompt1_mqa_single_correct_sk|0 -arc.prompt2_mqa_single_correct_sk|0 -belebele.prompt3_mqa_single_correct_sk|0 -belebele.prompt1_mqa_single_correct_sk|0 -belebele.prompt2_mqa_single_correct_sk|0 -financial_exams.prompt1_completion_sk|0 -financial_exams.prompt3_completion_sk|0 -financial_exams.prompt2_completion_sk|0 -mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt3_mqa_single_correct_sk:anatomy|0 -mmlu.prompt1_mqa_single_correct_sk:anatomy|0 -mmlu.prompt2_mqa_single_correct_sk:anatomy|0 -mmlu.prompt3_mqa_single_correct_sk:astronomy|0 -mmlu.prompt1_mqa_single_correct_sk:astronomy|0 -mmlu.prompt2_mqa_single_correct_sk:astronomy|0 -mmlu.prompt3_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt1_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt2_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt3_mqa_single_correct_sk:college_biology|0 -mmlu.prompt1_mqa_single_correct_sk:college_biology|0 -mmlu.prompt2_mqa_single_correct_sk:college_biology|0 -mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:college_physics|0 -mmlu.prompt1_mqa_single_correct_sk:college_physics|0 -mmlu.prompt2_mqa_single_correct_sk:college_physics|0 -mmlu.prompt3_mqa_single_correct_sk:computer_security|0 -mmlu.prompt1_mqa_single_correct_sk:computer_security|0 -mmlu.prompt2_mqa_single_correct_sk:computer_security|0 -mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt3_mqa_single_correct_sk:econometrics|0 -mmlu.prompt1_mqa_single_correct_sk:econometrics|0 -mmlu.prompt2_mqa_single_correct_sk:econometrics|0 -mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt1_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt2_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt3_mqa_single_correct_sk:global_facts|0 -mmlu.prompt1_mqa_single_correct_sk:global_facts|0 -mmlu.prompt2_mqa_single_correct_sk:global_facts|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt3_mqa_single_correct_sk:human_aging|0 -mmlu.prompt1_mqa_single_correct_sk:human_aging|0 -mmlu.prompt2_mqa_single_correct_sk:human_aging|0 -mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt3_mqa_single_correct_sk:international_law|0 -mmlu.prompt1_mqa_single_correct_sk:international_law|0 -mmlu.prompt2_mqa_single_correct_sk:international_law|0 -mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt3_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt1_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt2_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt3_mqa_single_correct_sk:management|0 -mmlu.prompt1_mqa_single_correct_sk:management|0 -mmlu.prompt2_mqa_single_correct_sk:management|0 -mmlu.prompt3_mqa_single_correct_sk:marketing|0 -mmlu.prompt1_mqa_single_correct_sk:marketing|0 -mmlu.prompt2_mqa_single_correct_sk:marketing|0 -mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt3_mqa_single_correct_sk:nutrition|0 -mmlu.prompt1_mqa_single_correct_sk:nutrition|0 -mmlu.prompt2_mqa_single_correct_sk:nutrition|0 -mmlu.prompt3_mqa_single_correct_sk:philosophy|0 -mmlu.prompt1_mqa_single_correct_sk:philosophy|0 -mmlu.prompt2_mqa_single_correct_sk:philosophy|0 -mmlu.prompt3_mqa_single_correct_sk:prehistory|0 -mmlu.prompt1_mqa_single_correct_sk:prehistory|0 -mmlu.prompt2_mqa_single_correct_sk:prehistory|0 -mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt3_mqa_single_correct_sk:professional_law|0 -mmlu.prompt1_mqa_single_correct_sk:professional_law|0 -mmlu.prompt2_mqa_single_correct_sk:professional_law|0 -mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:public_relations|0 -mmlu.prompt1_mqa_single_correct_sk:public_relations|0 -mmlu.prompt2_mqa_single_correct_sk:public_relations|0 -mmlu.prompt3_mqa_single_correct_sk:security_studies|0 -mmlu.prompt1_mqa_single_correct_sk:security_studies|0 -mmlu.prompt2_mqa_single_correct_sk:security_studies|0 -mmlu.prompt3_mqa_single_correct_sk:sociology|0 -mmlu.prompt1_mqa_single_correct_sk:sociology|0 -mmlu.prompt2_mqa_single_correct_sk:sociology|0 -mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt3_mqa_single_correct_sk:virology|0 -mmlu.prompt1_mqa_single_correct_sk:virology|0 -mmlu.prompt2_mqa_single_correct_sk:virology|0 -mmlu.prompt3_mqa_single_correct_sk:world_religions|0 -mmlu.prompt1_mqa_single_correct_sk:world_religions|0 -mmlu.prompt2_mqa_single_correct_sk:world_religions|0 -multi_wiki_qa.prompt3_qa_sk|0 -multi_wiki_qa.prompt2_qa_sk|0 -multi_wiki_qa.prompt1_qa_sk|0 -senti_sk.prompt2_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_binary_sk|0 -senti_sk.prompt1_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_3_class_sk|0 -senti_sk.prompt1_sentiment_3_class_sk|0 -senti_sk.prompt2_sentiment_3_class_sk|0 -skj1.prompt3_skj1_sk|0 -skj1.prompt1_skj1_sk|0 -skj1.prompt2_skj1_sk|0 -sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_hate_speech_sk:hate-speech|0 -sklep.prompt3_hate_speech_sk:hate-speech|0 -sklep.prompt2_hate_speech_sk:hate-speech|0 -sklep.prompt1_nli_3_classes_sk:nli|0 -sklep.prompt2_nli_3_classes_sk:nli|0 -sklep.prompt3_nli_3_classes_sk:nli|0 -sklep.prompt1_nli_binary_sk:rte|0 -sklep.prompt3_nli_binary_sk:rte|0 -sklep.prompt2_nli_binary_sk:rte|0 -sklep.prompt3_qa_sk:question-answering|0 -sklep.prompt2_qa_sk:question-answering|0 -sklep.prompt1_qa_sk:question-answering|0 -toxic_sk.prompt2_toxicity_sk|0 -toxic_sk.prompt3_toxicity_sk|0 -toxic_sk.prompt1_toxicity_sk|0 -truthfulqa.prompt3_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt1_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt2_mqa_single_correct_sk:mc1|0 -slovakcopa.prompt3_copa_sk|0 -slovakcopa.prompt1_copa_sk|0 -slovakcopa.prompt2_copa_sk|0 -movie_review.prompt3_sentiment_binary_sk|0 -movie_review.prompt2_sentiment_binary_sk|0 -movie_review.prompt1_sentiment_binary_sk|0 -globalmmlu-lite.prompt3_mqa_single_correct_sk|0 -globalmmlu-lite.prompt1_mqa_single_correct_sk|0 -globalmmlu-lite.prompt2_mqa_single_correct_sk|0 diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero.txt b/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero.txt deleted file mode 100644 index 154f511..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero.txt +++ /dev/null @@ -1,412 +0,0 @@ -arc.prompt3_mqa_single_correct_sk|0 -arc.prompt1_mqa_single_correct_sk|0 -arc.prompt2_mqa_single_correct_sk|0 -belebele.prompt3_mqa_single_correct_sk|0 -belebele.prompt1_mqa_single_correct_sk|0 -belebele.prompt2_mqa_single_correct_sk|0 -chiby.prompt2_chiby_sk|0 -chiby.prompt1_chiby_sk|0 -chiby.prompt3_chiby_sk|0 -financial_exams.prompt1_completion_sk|0 -financial_exams.prompt3_completion_sk|0 -financial_exams.prompt2_completion_sk|0 -ifeval_sk.ifeval|0 -mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt3_mqa_single_correct_sk:anatomy|0 -mmlu.prompt1_mqa_single_correct_sk:anatomy|0 -mmlu.prompt2_mqa_single_correct_sk:anatomy|0 -mmlu.prompt3_mqa_single_correct_sk:astronomy|0 -mmlu.prompt1_mqa_single_correct_sk:astronomy|0 -mmlu.prompt2_mqa_single_correct_sk:astronomy|0 -mmlu.prompt3_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt1_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt2_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt3_mqa_single_correct_sk:college_biology|0 -mmlu.prompt1_mqa_single_correct_sk:college_biology|0 -mmlu.prompt2_mqa_single_correct_sk:college_biology|0 -mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:college_physics|0 -mmlu.prompt1_mqa_single_correct_sk:college_physics|0 -mmlu.prompt2_mqa_single_correct_sk:college_physics|0 -mmlu.prompt3_mqa_single_correct_sk:computer_security|0 -mmlu.prompt1_mqa_single_correct_sk:computer_security|0 -mmlu.prompt2_mqa_single_correct_sk:computer_security|0 -mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt3_mqa_single_correct_sk:econometrics|0 -mmlu.prompt1_mqa_single_correct_sk:econometrics|0 -mmlu.prompt2_mqa_single_correct_sk:econometrics|0 -mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt1_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt2_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt3_mqa_single_correct_sk:global_facts|0 -mmlu.prompt1_mqa_single_correct_sk:global_facts|0 -mmlu.prompt2_mqa_single_correct_sk:global_facts|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt3_mqa_single_correct_sk:human_aging|0 -mmlu.prompt1_mqa_single_correct_sk:human_aging|0 -mmlu.prompt2_mqa_single_correct_sk:human_aging|0 -mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt3_mqa_single_correct_sk:international_law|0 -mmlu.prompt1_mqa_single_correct_sk:international_law|0 -mmlu.prompt2_mqa_single_correct_sk:international_law|0 -mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt3_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt1_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt2_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt3_mqa_single_correct_sk:management|0 -mmlu.prompt1_mqa_single_correct_sk:management|0 -mmlu.prompt2_mqa_single_correct_sk:management|0 -mmlu.prompt3_mqa_single_correct_sk:marketing|0 -mmlu.prompt1_mqa_single_correct_sk:marketing|0 -mmlu.prompt2_mqa_single_correct_sk:marketing|0 -mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt3_mqa_single_correct_sk:nutrition|0 -mmlu.prompt1_mqa_single_correct_sk:nutrition|0 -mmlu.prompt2_mqa_single_correct_sk:nutrition|0 -mmlu.prompt3_mqa_single_correct_sk:philosophy|0 -mmlu.prompt1_mqa_single_correct_sk:philosophy|0 -mmlu.prompt2_mqa_single_correct_sk:philosophy|0 -mmlu.prompt3_mqa_single_correct_sk:prehistory|0 -mmlu.prompt1_mqa_single_correct_sk:prehistory|0 -mmlu.prompt2_mqa_single_correct_sk:prehistory|0 -mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt3_mqa_single_correct_sk:professional_law|0 -mmlu.prompt1_mqa_single_correct_sk:professional_law|0 -mmlu.prompt2_mqa_single_correct_sk:professional_law|0 -mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:public_relations|0 -mmlu.prompt1_mqa_single_correct_sk:public_relations|0 -mmlu.prompt2_mqa_single_correct_sk:public_relations|0 -mmlu.prompt3_mqa_single_correct_sk:security_studies|0 -mmlu.prompt1_mqa_single_correct_sk:security_studies|0 -mmlu.prompt2_mqa_single_correct_sk:security_studies|0 -mmlu.prompt3_mqa_single_correct_sk:sociology|0 -mmlu.prompt1_mqa_single_correct_sk:sociology|0 -mmlu.prompt2_mqa_single_correct_sk:sociology|0 -mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt3_mqa_single_correct_sk:virology|0 -mmlu.prompt1_mqa_single_correct_sk:virology|0 -mmlu.prompt2_mqa_single_correct_sk:virology|0 -mmlu.prompt3_mqa_single_correct_sk:world_religions|0 -mmlu.prompt1_mqa_single_correct_sk:world_religions|0 -mmlu.prompt2_mqa_single_correct_sk:world_religions|0 -multi_wiki_qa.prompt3_qa_sk|0 -multi_wiki_qa.prompt2_qa_sk|0 -multi_wiki_qa.prompt1_qa_sk|0 -nli_sk.prompt1_nli_3_classes_sk|0 -nli_sk.prompt2_nli_3_classes_sk|0 -nli_sk.prompt3_nli_3_classes_sk|0 -oneruler_sk.oneruler_slovak:cwe_4096|0 -oneruler_sk.oneruler_slovak:cwe_8192|0 -oneruler_sk.oneruler_slovak:cwe_16384|0 -oneruler_sk.oneruler_slovak:niah_multikey_4096|0 -oneruler_sk.oneruler_slovak:niah_multikey_8192|0 -oneruler_sk.oneruler_slovak:niah_multikey_16384|0 -oneruler_sk.oneruler_slovak:niah_multiquery_4096|0 -oneruler_sk.oneruler_slovak:niah_multiquery_8192|0 -oneruler_sk.oneruler_slovak:niah_multiquery_16384|0 -oneruler_sk.oneruler_slovak:niah_multivalue_4096|0 -oneruler_sk.oneruler_slovak:niah_multivalue_8192|0 -oneruler_sk.oneruler_slovak:niah_multivalue_16384|0 -oneruler_sk.oneruler_slovak:niah_none_4096|0 -oneruler_sk.oneruler_slovak:niah_none_8192|0 -oneruler_sk.oneruler_slovak:niah_none_16384|0 -oneruler_sk.oneruler_slovak:niah_single_4096|0 -oneruler_sk.oneruler_slovak:niah_single_8192|0 -oneruler_sk.oneruler_slovak:niah_single_16384|0 -senti_sk.prompt2_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_binary_sk|0 -senti_sk.prompt1_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_3_class_sk|0 -senti_sk.prompt1_sentiment_3_class_sk|0 -senti_sk.prompt2_sentiment_3_class_sk|0 -skj1.prompt3_skj1_sk|0 -skj1.prompt1_skj1_sk|0 -skj1.prompt2_skj1_sk|0 -sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_hate_speech_sk:hate-speech|0 -sklep.prompt3_hate_speech_sk:hate-speech|0 -sklep.prompt2_hate_speech_sk:hate-speech|0 -sklep.prompt1_nli_3_classes_sk:nli|0 -sklep.prompt2_nli_3_classes_sk:nli|0 -sklep.prompt3_nli_3_classes_sk:nli|0 -sklep.prompt1_nli_binary_sk:rte|0 -sklep.prompt3_nli_binary_sk:rte|0 -sklep.prompt2_nli_binary_sk:rte|0 -sklep.prompt3_qa_sk:question-answering|0 -sklep.prompt2_qa_sk:question-answering|0 -sklep.prompt1_qa_sk:question-answering|0 -toxic_sk.prompt2_toxicity_sk|0 -toxic_sk.prompt3_toxicity_sk|0 -toxic_sk.prompt1_toxicity_sk|0 -truthfulqa.prompt3_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt1_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt2_mqa_single_correct_sk:mc1|0 -slovakcopa.prompt3_copa_sk|0 -slovakcopa.prompt1_copa_sk|0 -slovakcopa.prompt2_copa_sk|0 -slovaksum.prompt2_summarization_sk|0 -slovaksum.prompt3_summarization_sk|0 -slovaksum.prompt1_summarization_sk|0 -smesum.prompt2_summarization_sk|0 -smesum.prompt3_summarization_sk|0 -smesum.prompt1_summarization_sk|0 -movie_review.prompt3_sentiment_binary_sk|0 -movie_review.prompt2_sentiment_binary_sk|0 -movie_review.prompt1_sentiment_binary_sk|0 -slovak_trivia.prompt2_mqa_single_correct_sk|0 -slovak_trivia.prompt3_mqa_single_correct_sk|0 -slovak_trivia.prompt1_mqa_single_correct_sk|0 -analogies_sk.prompt1_analogy_sk|0 -analogies_sk.prompt2_analogy_sk|0 -analogies_sk.prompt3_analogy_sk|0 -driving_test.prompt3_mqa_single_correct_sk|0 -driving_test.prompt1_mqa_single_correct_sk|0 -driving_test.prompt2_mqa_single_correct_sk|0 -e_tests.prompt3_mqa_single_correct_sk:biologia|0 -e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt3_mqa_single_correct_sk:informatika|0 -e_tests.prompt3_mqa_single_correct_sk:geografia|0 -e_tests.prompt3_mqa_single_correct_sk:matematika|0 -e_tests.prompt3_mqa_single_correct_sk:chemia|0 -e_tests.prompt3_mqa_single_correct_sk:dejepis|0 -e_tests.prompt3_mqa_single_correct_sk:fyzika|0 -e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt1_mqa_single_correct_sk:biologia|0 -e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt1_mqa_single_correct_sk:informatika|0 -e_tests.prompt1_mqa_single_correct_sk:geografia|0 -e_tests.prompt1_mqa_single_correct_sk:matematika|0 -e_tests.prompt1_mqa_single_correct_sk:chemia|0 -e_tests.prompt1_mqa_single_correct_sk:dejepis|0 -e_tests.prompt1_mqa_single_correct_sk:fyzika|0 -e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt2_mqa_single_correct_sk:biologia|0 -e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt2_mqa_single_correct_sk:informatika|0 -e_tests.prompt2_mqa_single_correct_sk:geografia|0 -e_tests.prompt2_mqa_single_correct_sk:matematika|0 -e_tests.prompt2_mqa_single_correct_sk:chemia|0 -e_tests.prompt2_mqa_single_correct_sk:dejepis|0 -e_tests.prompt2_mqa_single_correct_sk:fyzika|0 -e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt2_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt2_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt2_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt2_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt2_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0 -e_tests.prompt1_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt1_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt1_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt1_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt1_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0 -e_tests.prompt3_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt3_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt3_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt3_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt3_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0 -slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt3_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0 -slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt1_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0 -slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt2_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0 -e_tests.prompt1_qa_sk:biologia|0 -e_tests.prompt1_qa_sk:obcianska-nauka|0 -e_tests.prompt1_qa_sk:informatika|0 -e_tests.prompt1_qa_sk:geografia|0 -e_tests.prompt1_qa_sk:chemia|0 -e_tests.prompt1_qa_sk:dejepis|0 -e_tests.prompt1_qa_sk:fyzika|0 -e_tests.prompt1_qa_sk:slovensky-jazyk|0 -e_tests.prompt3_qa_sk:biologia|0 -e_tests.prompt3_qa_sk:obcianska-nauka|0 -e_tests.prompt3_qa_sk:informatika|0 -e_tests.prompt3_qa_sk:geografia|0 -e_tests.prompt3_qa_sk:chemia|0 -e_tests.prompt3_qa_sk:dejepis|0 -e_tests.prompt3_qa_sk:fyzika|0 -e_tests.prompt3_qa_sk:slovensky-jazyk|0 -e_tests.prompt2_qa_sk:biologia|0 -e_tests.prompt2_qa_sk:obcianska-nauka|0 -e_tests.prompt2_qa_sk:informatika|0 -e_tests.prompt2_qa_sk:geografia|0 -e_tests.prompt2_qa_sk:chemia|0 -e_tests.prompt2_qa_sk:dejepis|0 -e_tests.prompt2_qa_sk:fyzika|0 -e_tests.prompt2_qa_sk:slovensky-jazyk|0 -slovak_tests.prompt1_qa_sk:monitor9|0 -slovak_tests.prompt1_qa_sk:maturita|0 -slovak_tests.prompt1_qa_sk:prijmacie_skusky|0 -slovak_tests.prompt3_qa_sk:monitor9|0 -slovak_tests.prompt3_qa_sk:maturita|0 -slovak_tests.prompt3_qa_sk:prijmacie_skusky|0 -slovak_tests.prompt2_qa_sk:monitor9|0 -slovak_tests.prompt2_qa_sk:maturita|0 -slovak_tests.prompt2_qa_sk:prijmacie_skusky|0 -slohry.prompt2_declension_sk:declension_high|0 -slohry.prompt2_declension_sk:declension_mid|0 -slohry.prompt2_declension_sk:declension_low|0 -slohry.prompt1_declension_sk:declension_high|0 -slohry.prompt1_declension_sk:declension_mid|0 -slohry.prompt1_declension_sk:declension_low|0 -slohry.prompt3_declension_sk:declension_high|0 -slohry.prompt3_declension_sk:declension_mid|0 -slohry.prompt3_declension_sk:declension_low|0 -slohry.prompt3_gradation_sk:gradation_high|0 -slohry.prompt3_gradation_sk:gradation_mid|0 -slohry.prompt3_gradation_sk:gradation_low|0 -slohry.prompt1_gradation_sk:gradation_high|0 -slohry.prompt1_gradation_sk:gradation_mid|0 -slohry.prompt1_gradation_sk:gradation_low|0 -slohry.prompt2_gradation_sk:gradation_high|0 -slohry.prompt2_gradation_sk:gradation_mid|0 -slohry.prompt2_gradation_sk:gradation_low|0 -slohry.prompt1_identity_form_sk:identity_form_high|0 -slohry.prompt1_identity_form_sk:identity_form_mid|0 -slohry.prompt1_identity_form_sk:identity_form_low|0 -slohry.prompt2_identity_form_sk:identity_form_high|0 -slohry.prompt2_identity_form_sk:identity_form_mid|0 -slohry.prompt2_identity_form_sk:identity_form_low|0 -slohry.prompt3_identity_form_sk:identity_form_high|0 -slohry.prompt3_identity_form_sk:identity_form_mid|0 -slohry.prompt3_identity_form_sk:identity_form_low|0 -slohry.prompt2_inflection_sk:inflection_high|0 -slohry.prompt2_inflection_sk:inflection_mid|0 -slohry.prompt2_inflection_sk:inflection_low|0 -slohry.prompt1_inflection_sk:inflection_high|0 -slohry.prompt1_inflection_sk:inflection_mid|0 -slohry.prompt1_inflection_sk:inflection_low|0 -slohry.prompt3_inflection_sk:inflection_high|0 -slohry.prompt3_inflection_sk:inflection_mid|0 -slohry.prompt3_inflection_sk:inflection_low|0 -scio.prompt2_mqa_single_correct_sk:aj|0 -scio.prompt2_mqa_single_correct_sk:mat|0 -scio.prompt2_mqa_single_correct_sk:bio|0 -scio.prompt2_mqa_single_correct_sk:che|0 -scio.prompt2_mqa_single_correct_sk:vsp|0 -scio.prompt3_mqa_single_correct_sk:aj|0 -scio.prompt3_mqa_single_correct_sk:mat|0 -scio.prompt3_mqa_single_correct_sk:bio|0 -scio.prompt3_mqa_single_correct_sk:che|0 -scio.prompt3_mqa_single_correct_sk:vsp|0 -scio.prompt1_mqa_single_correct_sk:aj|0 -scio.prompt1_mqa_single_correct_sk:mat|0 -scio.prompt1_mqa_single_correct_sk:bio|0 -scio.prompt1_mqa_single_correct_sk:che|0 -scio.prompt1_mqa_single_correct_sk:vsp|0 -globalmmlu-lite.prompt3_mqa_single_correct_sk|0 -globalmmlu-lite.prompt1_mqa_single_correct_sk|0 -globalmmlu-lite.prompt2_mqa_single_correct_sk|0 diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available.txt b/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available.txt deleted file mode 100644 index 81d49a3..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available.txt +++ /dev/null @@ -1,225 +0,0 @@ -arc.prompt3_mqa_single_correct_sk|0 -arc.prompt1_mqa_single_correct_sk|0 -arc.prompt2_mqa_single_correct_sk|0 -belebele.prompt3_mqa_single_correct_sk|0 -belebele.prompt1_mqa_single_correct_sk|0 -belebele.prompt2_mqa_single_correct_sk|0 -financial_exams.prompt1_completion_sk|0 -financial_exams.prompt3_completion_sk|0 -financial_exams.prompt2_completion_sk|0 -mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt3_mqa_single_correct_sk:anatomy|0 -mmlu.prompt1_mqa_single_correct_sk:anatomy|0 -mmlu.prompt2_mqa_single_correct_sk:anatomy|0 -mmlu.prompt3_mqa_single_correct_sk:astronomy|0 -mmlu.prompt1_mqa_single_correct_sk:astronomy|0 -mmlu.prompt2_mqa_single_correct_sk:astronomy|0 -mmlu.prompt3_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt1_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt2_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt3_mqa_single_correct_sk:college_biology|0 -mmlu.prompt1_mqa_single_correct_sk:college_biology|0 -mmlu.prompt2_mqa_single_correct_sk:college_biology|0 -mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:college_physics|0 -mmlu.prompt1_mqa_single_correct_sk:college_physics|0 -mmlu.prompt2_mqa_single_correct_sk:college_physics|0 -mmlu.prompt3_mqa_single_correct_sk:computer_security|0 -mmlu.prompt1_mqa_single_correct_sk:computer_security|0 -mmlu.prompt2_mqa_single_correct_sk:computer_security|0 -mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt3_mqa_single_correct_sk:econometrics|0 -mmlu.prompt1_mqa_single_correct_sk:econometrics|0 -mmlu.prompt2_mqa_single_correct_sk:econometrics|0 -mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt1_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt2_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt3_mqa_single_correct_sk:global_facts|0 -mmlu.prompt1_mqa_single_correct_sk:global_facts|0 -mmlu.prompt2_mqa_single_correct_sk:global_facts|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt3_mqa_single_correct_sk:human_aging|0 -mmlu.prompt1_mqa_single_correct_sk:human_aging|0 -mmlu.prompt2_mqa_single_correct_sk:human_aging|0 -mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt3_mqa_single_correct_sk:international_law|0 -mmlu.prompt1_mqa_single_correct_sk:international_law|0 -mmlu.prompt2_mqa_single_correct_sk:international_law|0 -mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt3_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt1_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt2_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt3_mqa_single_correct_sk:management|0 -mmlu.prompt1_mqa_single_correct_sk:management|0 -mmlu.prompt2_mqa_single_correct_sk:management|0 -mmlu.prompt3_mqa_single_correct_sk:marketing|0 -mmlu.prompt1_mqa_single_correct_sk:marketing|0 -mmlu.prompt2_mqa_single_correct_sk:marketing|0 -mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt3_mqa_single_correct_sk:nutrition|0 -mmlu.prompt1_mqa_single_correct_sk:nutrition|0 -mmlu.prompt2_mqa_single_correct_sk:nutrition|0 -mmlu.prompt3_mqa_single_correct_sk:philosophy|0 -mmlu.prompt1_mqa_single_correct_sk:philosophy|0 -mmlu.prompt2_mqa_single_correct_sk:philosophy|0 -mmlu.prompt3_mqa_single_correct_sk:prehistory|0 -mmlu.prompt1_mqa_single_correct_sk:prehistory|0 -mmlu.prompt2_mqa_single_correct_sk:prehistory|0 -mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt3_mqa_single_correct_sk:professional_law|0 -mmlu.prompt1_mqa_single_correct_sk:professional_law|0 -mmlu.prompt2_mqa_single_correct_sk:professional_law|0 -mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:public_relations|0 -mmlu.prompt1_mqa_single_correct_sk:public_relations|0 -mmlu.prompt2_mqa_single_correct_sk:public_relations|0 -mmlu.prompt3_mqa_single_correct_sk:security_studies|0 -mmlu.prompt1_mqa_single_correct_sk:security_studies|0 -mmlu.prompt2_mqa_single_correct_sk:security_studies|0 -mmlu.prompt3_mqa_single_correct_sk:sociology|0 -mmlu.prompt1_mqa_single_correct_sk:sociology|0 -mmlu.prompt2_mqa_single_correct_sk:sociology|0 -mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt3_mqa_single_correct_sk:virology|0 -mmlu.prompt1_mqa_single_correct_sk:virology|0 -mmlu.prompt2_mqa_single_correct_sk:virology|0 -mmlu.prompt3_mqa_single_correct_sk:world_religions|0 -mmlu.prompt1_mqa_single_correct_sk:world_religions|0 -mmlu.prompt2_mqa_single_correct_sk:world_religions|0 -multi_wiki_qa.prompt3_qa_sk|0 -multi_wiki_qa.prompt2_qa_sk|0 -multi_wiki_qa.prompt1_qa_sk|0 -nli_sk.prompt1_nli_3_classes_sk|0 -nli_sk.prompt2_nli_3_classes_sk|0 -nli_sk.prompt3_nli_3_classes_sk|0 -senti_sk.prompt2_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_binary_sk|0 -senti_sk.prompt1_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_3_class_sk|0 -senti_sk.prompt1_sentiment_3_class_sk|0 -senti_sk.prompt2_sentiment_3_class_sk|0 -skj1.prompt3_skj1_sk|0 -skj1.prompt1_skj1_sk|0 -skj1.prompt2_skj1_sk|0 -sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_hate_speech_sk:hate-speech|0 -sklep.prompt3_hate_speech_sk:hate-speech|0 -sklep.prompt2_hate_speech_sk:hate-speech|0 -sklep.prompt1_nli_3_classes_sk:nli|0 -sklep.prompt2_nli_3_classes_sk:nli|0 -sklep.prompt3_nli_3_classes_sk:nli|0 -sklep.prompt1_nli_binary_sk:rte|0 -sklep.prompt3_nli_binary_sk:rte|0 -sklep.prompt2_nli_binary_sk:rte|0 -sklep.prompt3_qa_sk:question-answering|0 -sklep.prompt2_qa_sk:question-answering|0 -sklep.prompt1_qa_sk:question-answering|0 -toxic_sk.prompt2_toxicity_sk|0 -toxic_sk.prompt3_toxicity_sk|0 -toxic_sk.prompt1_toxicity_sk|0 -truthfulqa.prompt3_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt1_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt2_mqa_single_correct_sk:mc1|0 -slovakcopa.prompt3_copa_sk|0 -slovakcopa.prompt1_copa_sk|0 -slovakcopa.prompt2_copa_sk|0 -movie_review.prompt3_sentiment_binary_sk|0 -movie_review.prompt2_sentiment_binary_sk|0 -movie_review.prompt1_sentiment_binary_sk|0 -globalmmlu-lite.prompt3_mqa_single_correct_sk|0 -globalmmlu-lite.prompt1_mqa_single_correct_sk|0 -globalmmlu-lite.prompt2_mqa_single_correct_sk|0 diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available_without_nli.txt b/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available_without_nli.txt deleted file mode 100644 index 45b85f6..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_available_without_nli.txt +++ /dev/null @@ -1,222 +0,0 @@ -arc.prompt3_mqa_single_correct_sk|0 -arc.prompt1_mqa_single_correct_sk|0 -arc.prompt2_mqa_single_correct_sk|0 -belebele.prompt3_mqa_single_correct_sk|0 -belebele.prompt1_mqa_single_correct_sk|0 -belebele.prompt2_mqa_single_correct_sk|0 -financial_exams.prompt1_completion_sk|0 -financial_exams.prompt3_completion_sk|0 -financial_exams.prompt2_completion_sk|0 -mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0 -mmlu.prompt3_mqa_single_correct_sk:anatomy|0 -mmlu.prompt1_mqa_single_correct_sk:anatomy|0 -mmlu.prompt2_mqa_single_correct_sk:anatomy|0 -mmlu.prompt3_mqa_single_correct_sk:astronomy|0 -mmlu.prompt1_mqa_single_correct_sk:astronomy|0 -mmlu.prompt2_mqa_single_correct_sk:astronomy|0 -mmlu.prompt3_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt1_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt2_mqa_single_correct_sk:business_ethics|0 -mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0 -mmlu.prompt3_mqa_single_correct_sk:college_biology|0 -mmlu.prompt1_mqa_single_correct_sk:college_biology|0 -mmlu.prompt2_mqa_single_correct_sk:college_biology|0 -mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:college_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:college_physics|0 -mmlu.prompt1_mqa_single_correct_sk:college_physics|0 -mmlu.prompt2_mqa_single_correct_sk:college_physics|0 -mmlu.prompt3_mqa_single_correct_sk:computer_security|0 -mmlu.prompt1_mqa_single_correct_sk:computer_security|0 -mmlu.prompt2_mqa_single_correct_sk:computer_security|0 -mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0 -mmlu.prompt3_mqa_single_correct_sk:econometrics|0 -mmlu.prompt1_mqa_single_correct_sk:econometrics|0 -mmlu.prompt2_mqa_single_correct_sk:econometrics|0 -mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0 -mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt1_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt2_mqa_single_correct_sk:formal_logic|0 -mmlu.prompt3_mqa_single_correct_sk:global_facts|0 -mmlu.prompt1_mqa_single_correct_sk:global_facts|0 -mmlu.prompt2_mqa_single_correct_sk:global_facts|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0 -mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0 -mmlu.prompt3_mqa_single_correct_sk:human_aging|0 -mmlu.prompt1_mqa_single_correct_sk:human_aging|0 -mmlu.prompt2_mqa_single_correct_sk:human_aging|0 -mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0 -mmlu.prompt3_mqa_single_correct_sk:international_law|0 -mmlu.prompt1_mqa_single_correct_sk:international_law|0 -mmlu.prompt2_mqa_single_correct_sk:international_law|0 -mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0 -mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0 -mmlu.prompt3_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt1_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt2_mqa_single_correct_sk:machine_learning|0 -mmlu.prompt3_mqa_single_correct_sk:management|0 -mmlu.prompt1_mqa_single_correct_sk:management|0 -mmlu.prompt2_mqa_single_correct_sk:management|0 -mmlu.prompt3_mqa_single_correct_sk:marketing|0 -mmlu.prompt1_mqa_single_correct_sk:marketing|0 -mmlu.prompt2_mqa_single_correct_sk:marketing|0 -mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0 -mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0 -mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0 -mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0 -mmlu.prompt3_mqa_single_correct_sk:nutrition|0 -mmlu.prompt1_mqa_single_correct_sk:nutrition|0 -mmlu.prompt2_mqa_single_correct_sk:nutrition|0 -mmlu.prompt3_mqa_single_correct_sk:philosophy|0 -mmlu.prompt1_mqa_single_correct_sk:philosophy|0 -mmlu.prompt2_mqa_single_correct_sk:philosophy|0 -mmlu.prompt3_mqa_single_correct_sk:prehistory|0 -mmlu.prompt1_mqa_single_correct_sk:prehistory|0 -mmlu.prompt2_mqa_single_correct_sk:prehistory|0 -mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0 -mmlu.prompt3_mqa_single_correct_sk:professional_law|0 -mmlu.prompt1_mqa_single_correct_sk:professional_law|0 -mmlu.prompt2_mqa_single_correct_sk:professional_law|0 -mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0 -mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0 -mmlu.prompt3_mqa_single_correct_sk:public_relations|0 -mmlu.prompt1_mqa_single_correct_sk:public_relations|0 -mmlu.prompt2_mqa_single_correct_sk:public_relations|0 -mmlu.prompt3_mqa_single_correct_sk:security_studies|0 -mmlu.prompt1_mqa_single_correct_sk:security_studies|0 -mmlu.prompt2_mqa_single_correct_sk:security_studies|0 -mmlu.prompt3_mqa_single_correct_sk:sociology|0 -mmlu.prompt1_mqa_single_correct_sk:sociology|0 -mmlu.prompt2_mqa_single_correct_sk:sociology|0 -mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0 -mmlu.prompt3_mqa_single_correct_sk:virology|0 -mmlu.prompt1_mqa_single_correct_sk:virology|0 -mmlu.prompt2_mqa_single_correct_sk:virology|0 -mmlu.prompt3_mqa_single_correct_sk:world_religions|0 -mmlu.prompt1_mqa_single_correct_sk:world_religions|0 -mmlu.prompt2_mqa_single_correct_sk:world_religions|0 -multi_wiki_qa.prompt3_qa_sk|0 -multi_wiki_qa.prompt2_qa_sk|0 -multi_wiki_qa.prompt1_qa_sk|0 -senti_sk.prompt2_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_binary_sk|0 -senti_sk.prompt1_sentiment_binary_sk|0 -senti_sk.prompt3_sentiment_3_class_sk|0 -senti_sk.prompt1_sentiment_3_class_sk|0 -senti_sk.prompt2_sentiment_3_class_sk|0 -skj1.prompt3_skj1_sk|0 -skj1.prompt1_skj1_sk|0 -skj1.prompt2_skj1_sk|0 -sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0 -sklep.prompt1_hate_speech_sk:hate-speech|0 -sklep.prompt3_hate_speech_sk:hate-speech|0 -sklep.prompt2_hate_speech_sk:hate-speech|0 -sklep.prompt1_nli_3_classes_sk:nli|0 -sklep.prompt2_nli_3_classes_sk:nli|0 -sklep.prompt3_nli_3_classes_sk:nli|0 -sklep.prompt1_nli_binary_sk:rte|0 -sklep.prompt3_nli_binary_sk:rte|0 -sklep.prompt2_nli_binary_sk:rte|0 -sklep.prompt3_qa_sk:question-answering|0 -sklep.prompt2_qa_sk:question-answering|0 -sklep.prompt1_qa_sk:question-answering|0 -toxic_sk.prompt2_toxicity_sk|0 -toxic_sk.prompt3_toxicity_sk|0 -toxic_sk.prompt1_toxicity_sk|0 -truthfulqa.prompt3_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt1_mqa_single_correct_sk:mc1|0 -truthfulqa.prompt2_mqa_single_correct_sk:mc1|0 -slovakcopa.prompt3_copa_sk|0 -slovakcopa.prompt1_copa_sk|0 -slovakcopa.prompt2_copa_sk|0 -movie_review.prompt3_sentiment_binary_sk|0 -movie_review.prompt2_sentiment_binary_sk|0 -movie_review.prompt1_sentiment_binary_sk|0 -globalmmlu-lite.prompt3_mqa_single_correct_sk|0 -globalmmlu-lite.prompt1_mqa_single_correct_sk|0 -globalmmlu-lite.prompt2_mqa_single_correct_sk|0 diff --git a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_unavailable.txt b/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_unavailable.txt deleted file mode 100644 index de2a1d0..0000000 --- a/aya_300k_lr_experiments/benchmarks/skbench/configs/tasks_sk_zero_unavailable.txt +++ /dev/null @@ -1,187 +0,0 @@ -chiby.prompt2_chiby_sk|0 -chiby.prompt1_chiby_sk|0 -chiby.prompt3_chiby_sk|0 -ifeval_sk.ifeval|0 -oneruler_sk.oneruler_slovak:cwe_4096|0 -oneruler_sk.oneruler_slovak:cwe_8192|0 -oneruler_sk.oneruler_slovak:cwe_16384|0 -oneruler_sk.oneruler_slovak:niah_multikey_4096|0 -oneruler_sk.oneruler_slovak:niah_multikey_8192|0 -oneruler_sk.oneruler_slovak:niah_multikey_16384|0 -oneruler_sk.oneruler_slovak:niah_multiquery_4096|0 -oneruler_sk.oneruler_slovak:niah_multiquery_8192|0 -oneruler_sk.oneruler_slovak:niah_multiquery_16384|0 -oneruler_sk.oneruler_slovak:niah_multivalue_4096|0 -oneruler_sk.oneruler_slovak:niah_multivalue_8192|0 -oneruler_sk.oneruler_slovak:niah_multivalue_16384|0 -oneruler_sk.oneruler_slovak:niah_none_4096|0 -oneruler_sk.oneruler_slovak:niah_none_8192|0 -oneruler_sk.oneruler_slovak:niah_none_16384|0 -oneruler_sk.oneruler_slovak:niah_single_4096|0 -oneruler_sk.oneruler_slovak:niah_single_8192|0 -oneruler_sk.oneruler_slovak:niah_single_16384|0 -slovaksum.prompt2_summarization_sk|0 -slovaksum.prompt3_summarization_sk|0 -slovaksum.prompt1_summarization_sk|0 -smesum.prompt2_summarization_sk|0 -smesum.prompt3_summarization_sk|0 -smesum.prompt1_summarization_sk|0 -slovak_trivia.prompt2_mqa_single_correct_sk|0 -slovak_trivia.prompt3_mqa_single_correct_sk|0 -slovak_trivia.prompt1_mqa_single_correct_sk|0 -analogies_sk.prompt1_analogy_sk|0 -analogies_sk.prompt2_analogy_sk|0 -analogies_sk.prompt3_analogy_sk|0 -driving_test.prompt3_mqa_single_correct_sk|0 -driving_test.prompt1_mqa_single_correct_sk|0 -driving_test.prompt2_mqa_single_correct_sk|0 -e_tests.prompt3_mqa_single_correct_sk:biologia|0 -e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt3_mqa_single_correct_sk:informatika|0 -e_tests.prompt3_mqa_single_correct_sk:geografia|0 -e_tests.prompt3_mqa_single_correct_sk:matematika|0 -e_tests.prompt3_mqa_single_correct_sk:chemia|0 -e_tests.prompt3_mqa_single_correct_sk:dejepis|0 -e_tests.prompt3_mqa_single_correct_sk:fyzika|0 -e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt1_mqa_single_correct_sk:biologia|0 -e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt1_mqa_single_correct_sk:informatika|0 -e_tests.prompt1_mqa_single_correct_sk:geografia|0 -e_tests.prompt1_mqa_single_correct_sk:matematika|0 -e_tests.prompt1_mqa_single_correct_sk:chemia|0 -e_tests.prompt1_mqa_single_correct_sk:dejepis|0 -e_tests.prompt1_mqa_single_correct_sk:fyzika|0 -e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt2_mqa_single_correct_sk:biologia|0 -e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0 -e_tests.prompt2_mqa_single_correct_sk:informatika|0 -e_tests.prompt2_mqa_single_correct_sk:geografia|0 -e_tests.prompt2_mqa_single_correct_sk:matematika|0 -e_tests.prompt2_mqa_single_correct_sk:chemia|0 -e_tests.prompt2_mqa_single_correct_sk:dejepis|0 -e_tests.prompt2_mqa_single_correct_sk:fyzika|0 -e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0 -e_tests.prompt2_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt2_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt2_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt2_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt2_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0 -e_tests.prompt1_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt1_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt1_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt1_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt1_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0 -e_tests.prompt3_mqa_multiple_correct_sk:biologia|0 -e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0 -e_tests.prompt3_mqa_multiple_correct_sk:informatika|0 -e_tests.prompt3_mqa_multiple_correct_sk:geografia|0 -e_tests.prompt3_mqa_multiple_correct_sk:matematika|0 -e_tests.prompt3_mqa_multiple_correct_sk:chemia|0 -e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0 -e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0 -e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0 -slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt3_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0 -slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt1_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0 -slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0 -slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0 -slovak_tests.prompt2_mqa_single_correct_sk:maturita|0 -slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0 -e_tests.prompt1_qa_sk:biologia|0 -e_tests.prompt1_qa_sk:obcianska-nauka|0 -e_tests.prompt1_qa_sk:informatika|0 -e_tests.prompt1_qa_sk:geografia|0 -e_tests.prompt1_qa_sk:chemia|0 -e_tests.prompt1_qa_sk:dejepis|0 -e_tests.prompt1_qa_sk:fyzika|0 -e_tests.prompt1_qa_sk:slovensky-jazyk|0 -e_tests.prompt3_qa_sk:biologia|0 -e_tests.prompt3_qa_sk:obcianska-nauka|0 -e_tests.prompt3_qa_sk:informatika|0 -e_tests.prompt3_qa_sk:geografia|0 -e_tests.prompt3_qa_sk:chemia|0 -e_tests.prompt3_qa_sk:dejepis|0 -e_tests.prompt3_qa_sk:fyzika|0 -e_tests.prompt3_qa_sk:slovensky-jazyk|0 -e_tests.prompt2_qa_sk:biologia|0 -e_tests.prompt2_qa_sk:obcianska-nauka|0 -e_tests.prompt2_qa_sk:informatika|0 -e_tests.prompt2_qa_sk:geografia|0 -e_tests.prompt2_qa_sk:chemia|0 -e_tests.prompt2_qa_sk:dejepis|0 -e_tests.prompt2_qa_sk:fyzika|0 -e_tests.prompt2_qa_sk:slovensky-jazyk|0 -slovak_tests.prompt1_qa_sk:monitor9|0 -slovak_tests.prompt1_qa_sk:maturita|0 -slovak_tests.prompt1_qa_sk:prijmacie_skusky|0 -slovak_tests.prompt3_qa_sk:monitor9|0 -slovak_tests.prompt3_qa_sk:maturita|0 -slovak_tests.prompt3_qa_sk:prijmacie_skusky|0 -slovak_tests.prompt2_qa_sk:monitor9|0 -slovak_tests.prompt2_qa_sk:maturita|0 -slovak_tests.prompt2_qa_sk:prijmacie_skusky|0 -slohry.prompt2_declension_sk:declension_high|0 -slohry.prompt2_declension_sk:declension_mid|0 -slohry.prompt2_declension_sk:declension_low|0 -slohry.prompt1_declension_sk:declension_high|0 -slohry.prompt1_declension_sk:declension_mid|0 -slohry.prompt1_declension_sk:declension_low|0 -slohry.prompt3_declension_sk:declension_high|0 -slohry.prompt3_declension_sk:declension_mid|0 -slohry.prompt3_declension_sk:declension_low|0 -slohry.prompt3_gradation_sk:gradation_high|0 -slohry.prompt3_gradation_sk:gradation_mid|0 -slohry.prompt3_gradation_sk:gradation_low|0 -slohry.prompt1_gradation_sk:gradation_high|0 -slohry.prompt1_gradation_sk:gradation_mid|0 -slohry.prompt1_gradation_sk:gradation_low|0 -slohry.prompt2_gradation_sk:gradation_high|0 -slohry.prompt2_gradation_sk:gradation_mid|0 -slohry.prompt2_gradation_sk:gradation_low|0 -slohry.prompt1_identity_form_sk:identity_form_high|0 -slohry.prompt1_identity_form_sk:identity_form_mid|0 -slohry.prompt1_identity_form_sk:identity_form_low|0 -slohry.prompt2_identity_form_sk:identity_form_high|0 -slohry.prompt2_identity_form_sk:identity_form_mid|0 -slohry.prompt2_identity_form_sk:identity_form_low|0 -slohry.prompt3_identity_form_sk:identity_form_high|0 -slohry.prompt3_identity_form_sk:identity_form_mid|0 -slohry.prompt3_identity_form_sk:identity_form_low|0 -slohry.prompt2_inflection_sk:inflection_high|0 -slohry.prompt2_inflection_sk:inflection_mid|0 -slohry.prompt2_inflection_sk:inflection_low|0 -slohry.prompt1_inflection_sk:inflection_high|0 -slohry.prompt1_inflection_sk:inflection_mid|0 -slohry.prompt1_inflection_sk:inflection_low|0 -slohry.prompt3_inflection_sk:inflection_high|0 -slohry.prompt3_inflection_sk:inflection_mid|0 -slohry.prompt3_inflection_sk:inflection_low|0 -scio.prompt2_mqa_single_correct_sk:aj|0 -scio.prompt2_mqa_single_correct_sk:mat|0 -scio.prompt2_mqa_single_correct_sk:bio|0 -scio.prompt2_mqa_single_correct_sk:che|0 -scio.prompt2_mqa_single_correct_sk:vsp|0 -scio.prompt3_mqa_single_correct_sk:aj|0 -scio.prompt3_mqa_single_correct_sk:mat|0 -scio.prompt3_mqa_single_correct_sk:bio|0 -scio.prompt3_mqa_single_correct_sk:che|0 -scio.prompt3_mqa_single_correct_sk:vsp|0 -scio.prompt1_mqa_single_correct_sk:aj|0 -scio.prompt1_mqa_single_correct_sk:mat|0 -scio.prompt1_mqa_single_correct_sk:bio|0 -scio.prompt1_mqa_single_correct_sk:che|0 -scio.prompt1_mqa_single_correct_sk:vsp|0